net,pkt: add chunked support As we don't know the length of the message we want to send to the other end, we send a chunk size before each message. In later versions, sending the wants might benefit from batching the lines together. Signed-off-by: Carlos Martín Nieto <carlos@cmartin.tk>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
diff --git a/src/netops.c b/src/netops.c
index 7d8a7b2..da24279 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -23,6 +23,7 @@
#include "common.h"
#include "netops.h"
+#include "posix.h"
void gitno_buffer_setup(gitno_buffer *buf, char *data, unsigned int len, int fd)
{
@@ -138,6 +139,7 @@ int gitno_send(GIT_SOCKET s, const char *msg, size_t len, int flags)
return off;
}
+
#ifdef GIT_WIN32
int gitno_close(GIT_SOCKET s)
{
@@ -150,6 +152,20 @@ int gitno_close(GIT_SOCKET s)
}
#endif
+int gitno_send_chunk_size(int s, size_t len)
+{
+ char str[8] = {0};
+ int ret;
+
+ ret = p_snprintf(str, sizeof(str), "%zx", len);
+ if (ret >= (int) sizeof(str)) {
+ return git__throw(GIT_ESHORTBUFFER, "Your number is too fucking big");
+ }
+
+ return gitno_send(s, str, ret, 0 /* TODO: MSG_MORE */);
+}
+
+
int gitno_select_in(gitno_buffer *buf, long int sec, long int usec)
{
fd_set fds;
diff --git a/src/netops.h b/src/netops.h
index 203df85..f9a8127 100644
--- a/src/netops.h
+++ b/src/netops.h
@@ -28,6 +28,7 @@ void gitno_consume_n(gitno_buffer *buf, size_t cons);
int gitno_connect(const char *host, const char *port);
int gitno_send(GIT_SOCKET s, const char *msg, size_t len, int flags);
int gitno_close(GIT_SOCKET s);
+int gitno_send_chunk_size(int s, size_t len);
int gitno_select_in(gitno_buffer *buf, long int sec, long int usec);
int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port);
diff --git a/src/pkt.c b/src/pkt.c
index 319a22e..a9ac4ad 100644
--- a/src/pkt.c
+++ b/src/pkt.c
@@ -16,6 +16,7 @@
#include "util.h"
#include "netops.h"
#include "posix.h"
+#include "buffer.h"
#include <ctype.h>
@@ -261,18 +262,25 @@ void git_pkt_free(git_pkt *pkt)
free(pkt);
}
-int git_pkt_send_flush(int s)
+int git_pkt_send_flush(int s, int chunked)
{
char flush[] = "0000";
+ int error;
+ if (chunked) {
+ error = gitno_send_chunk_size(s, strlen(flush));
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Failed to send chunk size");
+ }
return gitno_send(s, flush, strlen(flush), 0);
}
-static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, int fd)
+static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, int fd, int chunked)
{
char capstr[20]; /* Longer than we need */
char oid[GIT_OID_HEXSZ +1] = {0}, *cmd;
int error, len;
+ git_buf buf = GIT_BUF_INIT;
if (caps->ofs_delta)
strcpy(capstr, GIT_CAP_OFS_DELTA);
@@ -283,9 +291,13 @@ static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps,
return GIT_ENOMEM;
git_oid_fmt(oid, &head->oid);
- memset(cmd, 0x0, len + 1);
- p_snprintf(cmd, len + 1, "%04xwant %s%c%s\n", len, oid, 0, capstr);
- error = gitno_send(fd, cmd, len, 0);
+ git_buf_printf(&buf, "%04xwant %s%c%s\n", len, oid, 0, capstr);
+ if (chunked) {
+ error = gitno_send_chunk_size(fd, len);
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Failed to send first want chunk size");
+ }
+ error = gitno_send(fd, git_buf_cstr(&buf), len, 0);
free(cmd);
return error;
}
@@ -296,7 +308,7 @@ static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps,
*/
#define WANT_PREFIX "0032want "
-int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
+int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd, int chunked)
{
unsigned int i = 0;
int error = GIT_SUCCESS;
@@ -317,7 +329,7 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
break;
}
- error = send_want_with_caps(refs->heads[i], caps, fd);
+ error = send_want_with_caps(refs->heads[i], caps, fd, chunked);
if (error < GIT_SUCCESS)
return git__rethrow(error, "Failed to send want pkt with caps");
/* Increase it here so it's correct whether we run this or not */
@@ -331,13 +343,17 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
continue;
git_oid_fmt(buf + strlen(WANT_PREFIX), &head->oid);
+ if (chunked) {
+ error = gitno_send_chunk_size(fd, strlen(buf));
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Failed to send want chunk size");
+ }
error = gitno_send(fd, buf, strlen(buf), 0);
- if (error < 0) {
+ if (error < GIT_SUCCESS)
return git__rethrow(error, "Failed to send want pkt");
- }
}
- return git_pkt_send_flush(fd);
+ return git_pkt_send_flush(fd, chunked);
}
/*
@@ -346,17 +362,29 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
*/
#define HAVE_PREFIX "0032have "
-int git_pkt_send_have(git_oid *oid, int fd)
+int git_pkt_send_have(git_oid *oid, int fd, int chunked)
{
char buf[] = "0032have 0000000000000000000000000000000000000000\n";
+ int error;
+ if (chunked) {
+ error = gitno_send_chunk_size(fd, strlen(buf));
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Failed to send chunk size");
+ }
git_oid_fmt(buf + strlen(HAVE_PREFIX), oid);
return gitno_send(fd, buf, strlen(buf), 0);
}
-int git_pkt_send_done(int fd)
+int git_pkt_send_done(int fd, int chunked)
{
char buf[] = "0009done\n";
+ int error;
+ if (chunked) {
+ error = gitno_send_chunk_size(fd, strlen(buf));
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Failed to send chunk size");
+ }
return gitno_send(fd, buf, strlen(buf), 0);
}
diff --git a/src/pkt.h b/src/pkt.h
index f4ed81c..0689e29 100644
--- a/src/pkt.h
+++ b/src/pkt.h
@@ -63,10 +63,10 @@ typedef struct {
} git_pkt_comment;
int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
-int git_pkt_send_flush(int s);
-int git_pkt_send_done(int s);
-int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd);
-int git_pkt_send_have(git_oid *oid, int fd);
+int git_pkt_send_flush(int s, int chunked);
+int git_pkt_send_done(int s, int chunked);
+int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd, int chunked);
+int git_pkt_send_have(git_oid *oid, int fd, int chunked);
void git_pkt_free(git_pkt *pkt);
#endif
diff --git a/src/transport_git.c b/src/transport_git.c
index b8a4137..d3d5747 100644
--- a/src/transport_git.c
+++ b/src/transport_git.c
@@ -269,14 +269,14 @@ static int git_send_wants(git_transport *transport, git_headarray *array)
{
transport_git *t = (transport_git *) transport;
- return git_pkt_send_wants(array, &t->caps, t->socket);
+ return git_pkt_send_wants(array, &t->caps, t->socket, 0);
}
static int git_send_have(git_transport *transport, git_oid *oid)
{
transport_git *t = (transport_git *) transport;
- return git_pkt_send_have(oid, t->socket);
+ return git_pkt_send_have(oid, t->socket, 0);
}
static int git_negotiate_fetch(git_transport *transport, git_repository *repo, git_headarray *GIT_UNUSED(list))
@@ -333,12 +333,12 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, g
*/
i = 0;
while ((error = git_revwalk_next(&oid, walk)) == GIT_SUCCESS) {
- error = git_pkt_send_have(&oid, t->socket);
+ error = git_pkt_send_have(&oid, t->socket, 1);
i++;
if (i % 20 == 0) {
const char *ptr = buf.data, *line_end;
git_pkt *pkt;
- git_pkt_send_flush(t->socket);
+ git_pkt_send_flush(t->socket, 0);
while (1) {
/* Wait for max. 1 second */
error = gitno_select_in(&buf, 1, 0);
@@ -384,8 +384,8 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, g
error = GIT_SUCCESS;
done:
- git_pkt_send_flush(t->socket);
- git_pkt_send_done(t->socket);
+ git_pkt_send_flush(t->socket, 0);
+ git_pkt_send_done(t->socket, 0);
cleanup:
git_revwalk_free(walk);
@@ -396,14 +396,14 @@ static int git_send_flush(git_transport *transport)
{
transport_git *t = (transport_git *) transport;
- return git_pkt_send_flush(t->socket);
+ return git_pkt_send_flush(t->socket, 1);
}
static int git_send_done(git_transport *transport)
{
transport_git *t = (transport_git *) transport;
- return git_pkt_send_done(t->socket);
+ return git_pkt_send_done(t->socket, 1);
}
static int store_pack(char **out, gitno_buffer *buf, git_repository *repo)
@@ -502,9 +502,10 @@ static int git_close(git_transport *transport)
transport_git *t = (transport_git*) transport;
int error;
- /* Can't do anything if there's an error, so don't bother checking */
- git_pkt_send_flush(t->socket);
+ /* Can't do anything if there's an error, so don't bother checking */
+ git_pkt_send_flush(t->socket, 0);
error = gitno_close(t->socket);
+
if (error < 0)
error = git__throw(GIT_EOSERR, "Failed to close socket");