http: download pack when fetching Unfortunately, we can't use the function in fetch.c due to chunked encoding and keep-alive connections. Signed-off-by: Carlos Martín Nieto <carlos@cmartin.tk>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
diff --git a/src/transport-http.c b/src/transport-http.c
index 9ec9d7a..13a95db 100644
--- a/src/transport-http.c
+++ b/src/transport-http.c
@@ -15,6 +15,9 @@
#include "buffer.h"
#include "pkt.h"
#include "refs.h"
+#include "fetch.h"
+#include "filebuf.h"
+#include "repository.h"
enum last_cb {
NONE,
@@ -314,7 +317,7 @@ static int http_connect(git_transport *transport, int direction)
goto cleanup;
}
- error = gitno_send(t->socket, git_buf_cstr(&request), strlen(git_buf_cstr(&request)), 0);
+ error = gitno_send(t->socket, request.ptr, request.size, 0);
if (error < GIT_SUCCESS)
error = git__rethrow(error, "Failed to send the HTTP request");
@@ -360,7 +363,6 @@ static int on_body_parse_response(http_parser *parser, const char *str, size_t l
git_vector *common = &t->common;
int error;
const char *line_end, *ptr;
- static int first_pkt = 1;
if (len == 0) { /* EOF */
if (buf->size != 0)
@@ -378,19 +380,14 @@ static int on_body_parse_response(http_parser *parser, const char *str, size_t l
return 0;
error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->size);
- if (error == GIT_ESHORTBUFFER)
+ if (error == GIT_ESHORTBUFFER) {
return 0; /* Ask for more */
+ }
if (error < GIT_SUCCESS)
return t->error = git__rethrow(error, "Failed to parse pkt-line");
git_buf_consume(buf, line_end);
- if (first_pkt) {
- first_pkt = 0;
- if (pkt->type != GIT_PKT_COMMENT)
- return t->error = git__throw(GIT_EOBJCORRUPTED, "Not a valid smart HTTP response");
- }
-
if (pkt->type == GIT_PKT_PACK) {
t->pack_ready = 1;
return 0;
@@ -420,6 +417,7 @@ static int parse_response(transport_http *t)
http_parser_init(&t->parser, HTTP_RESPONSE);
t->parser.data = t;
+ t->transfer_finished = 0;
memset(&settings, 0x0, sizeof(http_parser_settings));
settings.on_header_field = on_header_field;
settings.on_header_value = on_header_value;
@@ -443,8 +441,9 @@ static int parse_response(transport_http *t)
gitno_consume_n(&buf, parsed);
- if (error == 0 || t->transfer_finished || t->pack_ready)
+ if (error == 0 || t->transfer_finished || t->pack_ready) {
return GIT_SUCCESS;
+ }
}
return error;
@@ -584,9 +583,15 @@ static int http_negotiate_fetch(git_transport *transport, git_repository *repo,
break;
error = parse_response(t);
+ if (error < GIT_SUCCESS) {
+ error = git__rethrow(error, "Error parsing the response");
+ goto cleanup;
+ }
- if (t->pack_ready)
- return 0;
+ if (t->pack_ready) {
+ error = GIT_SUCCESS;
+ goto cleanup;
+ }
} while(1);
@@ -595,6 +600,111 @@ cleanup:
return error;
}
+typedef struct {
+ git_filebuf *file;
+ transport_http *transport;
+} download_pack_cbdata;
+
+static int on_message_complete_download_pack(http_parser *parser)
+{
+ download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
+
+ data->transport->transfer_finished = 1;
+
+ return 0;
+}
+static int on_body_download_pack(http_parser *parser, const char *str, size_t len)
+{
+ download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
+ transport_http *t = data->transport;
+ git_filebuf *file = data->file;
+
+
+ return t->error = git_filebuf_write(file, str, len);
+}
+
+/*
+ * As the server is probably using Transfer-Encoding: chunked, we have
+ * to use the HTTP parser to download the pack instead of giving it to
+ * the simple downloader. Furthermore, we're using keep-alive
+ * connections, so the simple downloader would just hang.
+ */
+static int http_download_pack(char **out, git_transport *transport, git_repository *repo)
+{
+ transport_http *t = (transport_http *) transport;
+ git_buf *oldbuf = &t->buf;
+ int error = GIT_SUCCESS;
+ http_parser_settings settings;
+ char buffer[1024];
+ gitno_buffer buf;
+ download_pack_cbdata data;
+ git_filebuf file;
+ char path[GIT_PATH_MAX], suff[] = "/objects/pack/pack-received\0";
+
+ /*
+ * This is part of the previous response, so we don't want to
+ * re-init the parser, just set these two callbacks.
+ */
+ data.file = &file;
+ data.transport = t;
+ t->parser.data = &data;
+ t->transfer_finished = 0;
+ memset(&settings, 0x0, sizeof(settings));
+ settings.on_message_complete = on_message_complete_download_pack;
+ settings.on_body = on_body_download_pack;
+
+ gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket);
+
+ git_path_join(path, repo->path_repository, suff);
+
+ if (memcmp(oldbuf->ptr, "PACK", strlen("PACK"))) {
+ return git__throw(GIT_ERROR, "The pack doesn't start with the signature");
+ }
+
+ error = git_filebuf_open(&file, path, GIT_FILEBUF_TEMPORARY);
+ if (error < GIT_SUCCESS)
+ goto cleanup;
+
+ /* Part of the packfile has been received, don't loose it */
+ error = git_filebuf_write(&file, oldbuf->ptr, oldbuf->size);
+ if (error < GIT_SUCCESS)
+ goto cleanup;
+
+ while(1) {
+ size_t parsed;
+
+ error = gitno_recv(&buf);
+ if (error < GIT_SUCCESS)
+ return git__rethrow(error, "Error receiving data from network");
+
+ parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
+ /* Both should happen at the same time */
+ if (parsed != buf.offset || t->error < GIT_SUCCESS)
+ return git__rethrow(t->error, "Error parsing HTTP data");
+
+ gitno_consume_n(&buf, parsed);
+
+ if (error == 0 || t->transfer_finished) {
+ break;
+ }
+ }
+
+ *out = git__strdup(file.path_lock);
+ if (*out == NULL) {
+ error = GIT_ENOMEM;
+ goto cleanup;
+ }
+
+ /* A bit dodgy, but we need to keep the pack at the temporary path */
+ error = git_filebuf_commit_at(&file, file.path_lock);
+
+cleanup:
+ if (error < GIT_SUCCESS)
+ git_filebuf_cleanup(&file);
+
+ return error;
+}
+
static int http_close(git_transport *transport)
{
transport_http *t = (transport_http *) transport;
@@ -650,6 +760,7 @@ int git_transport_http(git_transport **out)
t->parent.connect = http_connect;
t->parent.ls = http_ls;
t->parent.negotiate_fetch = http_negotiate_fetch;
+ t->parent.download_pack = http_download_pack;
t->parent.close = http_close;
t->parent.free = http_free;