thodg/got

diff --git a/got/got.c b/got/got.c
index 741cccf..a3cf252 100644
--- a/got/got.c
+++ b/got/got.c
@@ -969,6 +969,15 @@ done:
 }
 
 static const struct got_error *
+fetch_progress(void *arg, const char *message)
+{
+	char *servername = arg;
+	printf("\rserver %s: %s", servername, message);
+	fflush(stdout);
+	return NULL;
+}
+
+static const struct got_error *
 cmd_clone(int argc, char *argv[])
 {
 	const struct got_error *err = NULL;
@@ -1036,7 +1045,7 @@ cmd_clone(int argc, char *argv[])
 		goto done;
 
 	err = got_fetch_pack(&pack_hash, &refs, &symrefs, fetchfd,
-	    repo);
+	    repo, fetch_progress, host);
 	if (err)
 		goto done;
 
diff --git a/include/got_fetch.h b/include/got_fetch.h
index 36345e8..e3e0144 100644
--- a/include/got_fetch.h
+++ b/include/got_fetch.h
@@ -38,6 +38,10 @@ const struct got_error *got_fetch_parse_uri(char **, char **, char **,
 const struct got_error *got_fetch_connect(int *, const char *, const char *,
     const char *, const char *);
 
+/* A callback function which gets invoked with progress information to print. */
+typedef const struct got_error *(*got_fetch_progress_cb)(void *,
+    const char *);
+
 /*
  * Attempt to fetch a packfile from a server. This pack file will contain
  * objects which that are not yet contained in the provided repository.
@@ -46,4 +50,4 @@ const struct got_error *got_fetch_connect(int *, const char *, const char *,
  */
 const struct got_error *got_fetch_pack(struct got_object_id **,
 	struct got_pathlist_head *, struct got_pathlist_head *, int,
-	struct got_repository *);
+	struct got_repository *, got_fetch_progress_cb, void *);
diff --git a/lib/fetch.c b/lib/fetch.c
index 4016f64..15ecb2b 100644
--- a/lib/fetch.c
+++ b/lib/fetch.c
@@ -299,7 +299,8 @@ done:
 
 const struct got_error*
 got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
-    struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo)
+    struct got_pathlist_head *symrefs, int fetchfd, struct got_repository *repo,
+    got_fetch_progress_cb progress_cb, void *progress_arg)
 {
 	int imsg_fetchfds[2], imsg_idxfds[2];
 	int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1;
@@ -387,9 +388,10 @@ got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 	while (!done) {
 		struct got_object_id *id = NULL;
 		char *refname = NULL;
+		char *server_progress = NULL;
 
 		err = got_privsep_recv_fetch_progress(&done,
-		    &id, &refname, symrefs, &ibuf);
+		    &id, &refname, symrefs, &server_progress, &ibuf);
 		if (err != NULL)
 			goto done;
 		if (done)
@@ -398,14 +400,25 @@ got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 			err = got_pathlist_append(refs, refname, id);
 			if (err)
 				goto done;
+		} else if (server_progress) {
+			char *s, *s0 = server_progress;
+			while ((s = strsep(&s0, "\r")) != NULL) {
+				if (*s == '\0')
+					continue;
+				err = progress_cb(progress_arg, s);
+				if (err)
+					break;
+			}
+			free(server_progress);
+			if (err)
+				goto done;
 		}
-		/* TODO remote status / download progress callback */
 	}
 	if (waitpid(pid, &status, 0) == -1) {
 		err = got_error_from_errno("waitpid");
 		goto done;
 	}
-
+ 
 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 		err = got_error_from_errno("socketpair");
 		goto done;
diff --git a/lib/got_lib_privsep.h b/lib/got_lib_privsep.h
index 05435e3..9b2faf4 100644
--- a/lib/got_lib_privsep.h
+++ b/lib/got_lib_privsep.h
@@ -112,6 +112,7 @@ enum got_imsg_type {
 	GOT_IMSG_FETCH_REQUEST,
 	GOT_IMSG_FETCH_SYMREFS,
 	GOT_IMSG_FETCH_PROGRESS,
+	GOT_IMSG_FETCH_SERVER_PROGRESS,
 	GOT_IMSG_FETCH_DONE,
 	GOT_IMSG_IDXPACK_REQUEST,
 	GOT_IMSG_IDXPACK_DONE,
@@ -356,9 +357,11 @@ const struct got_error *got_privsep_send_fetch_symrefs(struct imsgbuf *,
     struct got_pathlist_head *);
 const struct got_error *got_privsep_send_fetch_progress(struct imsgbuf *,
     struct got_object_id *, const char *);
+const struct got_error *got_privsep_send_fetch_server_progress(struct imsgbuf *,
+    const char *, size_t);
 const struct got_error *got_privsep_recv_fetch_progress(int *,
     struct got_object_id **, char **, struct got_pathlist_head *,
-    struct imsgbuf *);
+    char **, struct imsgbuf *);
 const struct got_error *got_privsep_send_fetch_done(struct imsgbuf *,
     struct got_object_id);
 const struct got_error *got_privsep_get_imsg_obj(struct got_object **,
diff --git a/lib/privsep.c b/lib/privsep.c
index 6bd3dc9..bbd1527 100644
--- a/lib/privsep.c
+++ b/lib/privsep.c
@@ -21,6 +21,7 @@
 #include <sys/syslimits.h>
 #include <sys/wait.h>
 
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -569,6 +570,24 @@ got_privsep_send_fetch_progress(struct imsgbuf *ibuf,
 }
 
 const struct got_error *
+got_privsep_send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg,
+    size_t msglen)
+{
+	if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+		return got_error(GOT_ERR_NO_SPACE);
+
+	if (msglen == 0)
+		return NULL;
+
+	if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
+	    msg, msglen) == -1)
+		return got_error_from_errno(
+		    "imsg_compose FETCH_SERVER_PROGRESS");
+
+	return flush_imsg(ibuf);
+}
+
+const struct got_error *
 got_privsep_send_fetch_done(struct imsgbuf *ibuf, struct got_object_id hash)
 {
 	if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
@@ -580,30 +599,33 @@ got_privsep_send_fetch_done(struct imsgbuf *ibuf, struct got_object_id hash)
 
 const struct got_error *
 got_privsep_recv_fetch_progress(int *done, struct got_object_id **id,
-    char **refname, struct got_pathlist_head *symrefs, struct imsgbuf *ibuf)
+    char **refname, struct got_pathlist_head *symrefs, char **server_progress,
+    struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	struct imsg imsg;
 	size_t datalen;
-	const size_t min_datalen =
-	    MIN(MIN(sizeof(struct got_imsg_error),
-	    sizeof(struct got_imsg_fetch_progress)),
-	    sizeof(struct got_imsg_fetch_symrefs));
 	struct got_imsg_fetch_symrefs *isymrefs = NULL;
 	size_t n, remain;
 	off_t off;
+	int i;
 
 	*done = 0;
 	*id = NULL;
 	*refname = NULL;
+	*server_progress = NULL;
 
-	err = got_privsep_recv_imsg(&imsg, ibuf, min_datalen);
+	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
 	if (err)
 		return err;
 
 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
 	switch (imsg.hdr.type) {
 	case GOT_IMSG_ERROR:
+		if (datalen < sizeof(struct got_imsg_error)) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
 		err = recv_imsg_error(&imsg, datalen);
 		break;
 	case GOT_IMSG_FETCH_SYMREFS:
@@ -661,16 +683,16 @@ got_privsep_recv_fetch_progress(int *done, struct got_object_id **id,
 		}
 		break;
 	case GOT_IMSG_FETCH_PROGRESS:
+		if (datalen <= SHA1_DIGEST_LENGTH) {
+			err = got_error(GOT_ERR_PRIVSEP_MSG);
+			break;
+		}
 		*id = malloc(sizeof(**id));
 		if (*id == NULL) {
 			err = got_error_from_errno("malloc");
 			break;
 		}
 		memcpy((*id)->sha1, imsg.data, SHA1_DIGEST_LENGTH);
-		if (datalen <= SHA1_DIGEST_LENGTH) {
-			err = got_error(GOT_ERR_PRIVSEP_MSG);
-			break;
-		}
 		*refname = strndup(imsg.data + SHA1_DIGEST_LENGTH,
 		    datalen - SHA1_DIGEST_LENGTH);
 		if (*refname == NULL) {
@@ -678,6 +700,26 @@ got_privsep_recv_fetch_progress(int *done, struct got_object_id **id,
 			break;
 		}
 		break;
+	case GOT_IMSG_FETCH_SERVER_PROGRESS:
+		if (datalen == 0) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		*server_progress = strndup(imsg.data, datalen);
+		if (*server_progress == NULL) {
+			err = got_error_from_errno("strndup");
+			break;
+		}
+		for (i = 0; i < datalen; i++) {
+			if (!isprint((unsigned char)(*server_progress)[i]) &&
+			    !isspace((unsigned char)(*server_progress)[i])) {
+				err = got_error(GOT_ERR_PRIVSEP_MSG);
+				free(*server_progress);
+				*server_progress = NULL;
+				goto done;
+			}
+		}
+		break;
 	case GOT_IMSG_FETCH_DONE:
 		*id = malloc(sizeof(**id));
 		if (*id == NULL) {
diff --git a/libexec/got-fetch-pack/got-fetch-pack.c b/libexec/got-fetch-pack/got-fetch-pack.c
index d74463f..b1b439e 100644
--- a/libexec/got-fetch-pack/got-fetch-pack.c
+++ b/libexec/got-fetch-pack/got-fetch-pack.c
@@ -91,58 +91,75 @@ flushpkt(int fd)
 	return NULL;
 }
 
-
+/*
+ * Packet header contains a 4-byte hexstring which specifies the length
+ * of data which follows.
+ */
 static const struct got_error *
-readpkt(int *outlen, int fd, char *buf, int nbuf)
+read_pkthdr(int *datalen, int fd)
 {
-	const struct got_error *err = NULL;
+	static const struct got_error *err = NULL;
 	char lenstr[5];
 	long len;
 	char *e;
 	int n, i;
 	ssize_t r;
 
-	*outlen = 0;
+	*datalen = 0;
 
 	err = readn(&r, fd, lenstr, 4);
 	if (err)
 		return err;
+	if (r == 0) /* implicit "0000" */
+		return NULL;
 	if (r != 4)
-		return got_error(GOT_ERR_IO);
+		return got_error_msg(GOT_ERR_BAD_PACKET,
+		    "wrong packet header length");
 
 	lenstr[4] = '\0';
 	for (i = 0; i < 4; i++) {
 		if (!isxdigit(lenstr[i]))
-			return got_error(GOT_ERR_BAD_PACKET);
+			return got_error_msg(GOT_ERR_BAD_PACKET,
+			    "packet length not specified in hex");
 	}
 	errno = 0;
 	len = strtol(lenstr, &e, 16);
 	if (lenstr[0] == '\0' || *e != '\0')
 		return got_error(GOT_ERR_BAD_PACKET);
 	if (errno == ERANGE && (len == LONG_MAX || len == LONG_MIN))
-		return got_error(GOT_ERR_BAD_PACKET);
+		return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
 	if (len > INT_MAX || len < INT_MIN)
-		return got_error(GOT_ERR_BAD_PACKET);
+		return got_error_msg(GOT_ERR_BAD_PACKET, "bad packet length");
 	n = len;
-	if (n == 0) {
-		if (chattygit)
-			fprintf(stderr, "readpkt: 0000\n");
+	if (n == 0)
 		return NULL;
-	}
 	if (n <= 4)
-		return got_error(GOT_ERR_BAD_PACKET);
+		return got_error_msg(GOT_ERR_BAD_PACKET, "packet too short");
 	n  -= 4;
-	if (n >= nbuf)
+
+	*datalen = n;
+	return NULL;
+}
+
+static const struct got_error *
+readpkt(int *outlen, int fd, char *buf, int buflen)
+{
+	const struct got_error *err = NULL;
+	int datalen;
+	ssize_t n;
+
+	err = read_pkthdr(&datalen, fd);
+	if (err)
+		return err;
+
+	if (datalen > buflen)
 		return got_error(GOT_ERR_NO_SPACE);
 
-	err = readn(&r, fd, buf, n);
+	err = readn(&n, fd, buf, datalen);
 	if (err)
 		return err;
-	if (r != n)
-		return got_error(GOT_ERR_BAD_PACKET);
-	buf[n] = 0;
-	if (chattygit)
-		fprintf(stderr, "readpkt: %s:\t%.*s\n", lenstr, nbuf, buf);
+	if (n != datalen)
+		return got_error_msg(GOT_ERR_BAD_PACKET, "short packet");
 
 	*outlen = n;
 	return NULL;
@@ -320,13 +337,23 @@ parse_refline(char **id_str, char **refname, char **server_capabilities,
 	return NULL;
 }
 
+#define GOT_CAPA_AGENT			"agent"
+#define GOT_CAPA_OFS_DELTA		"ofs-delta"
+#define GOT_CAPA_SIDE_BAND_64K		"side-band-64k"
+
+#define GOT_SIDEBAND_PACKFILE_DATA	1
+#define GOT_SIDEBAND_PROGRESS_INFO	2
+#define GOT_SIDEBAND_ERROR_INFO		3
+
+
 struct got_capability {
 	const char *key;
 	const char *value;
 };
 static const struct got_capability got_capabilities[] = {
-	{ "ofs-delta", NULL },
-	{ "agent", "got/" GOT_VERSION_STR },
+	{ GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
+	{ GOT_CAPA_OFS_DELTA, NULL },
+	{ GOT_CAPA_SIDE_BAND_64K, NULL },
 };
 
 static const struct got_error *
@@ -428,6 +455,50 @@ match_capabilities(char **my_capabilities, struct got_pathlist_head *symrefs,
 }
 
 static const struct got_error *
+fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
+{
+	int i;
+
+
+	if (len == 0)
+		return NULL;
+
+	/*
+	 * Truncate messages which exceed the maximum imsg payload size.
+	 * Server may send up to 64k.
+	 */
+	if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+		len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
+
+	/* Only allow printable ASCII. */
+	for (i = 0; i < len; i++) {
+		if (isprint((unsigned char)buf[i]) ||
+		    isspace((unsigned char)buf[i]))
+			continue;
+		return got_error_msg(GOT_ERR_BAD_PACKET,
+		    "non-printable progress message received from server");
+	}
+
+	return got_privsep_send_fetch_server_progress(ibuf, buf, len);
+}
+
+static const struct got_error *
+fetch_error(const char *buf, size_t len)
+{
+	static char msg[1024];
+	int i;
+
+	for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
+		if (!isprint(buf[i]))
+			return got_error_msg(GOT_ERR_BAD_PACKET,
+			    "non-printable error message received from server");
+		msg[i] = buf[i];
+	}
+	msg[i] = '\0';
+	return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
+}
+
+static const struct got_error *
 fetch_pack(int fd, int packfd, struct got_object_id *packid,
     struct got_pathlist_head *have_refs, struct imsgbuf *ibuf)
 {
@@ -442,6 +513,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *packid,
 	char *server_capabilities = NULL, *my_capabilities = NULL;
 	struct got_pathlist_head symrefs;
 	struct got_pathlist_entry *pe;
+	int have_sidebands = 0;
 
 	TAILQ_INIT(&symrefs);
 
@@ -462,16 +534,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *packid,
 		if (n == 0)
 			break;
 		if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
-			static char msg[1024];
-			for (i = 0; i < n && i < sizeof(msg) - 1; i++) {
-				if (!isprint(buf[i])) {
-					err = got_error(GOT_ERR_FETCH_FAILED);
-					goto done;
-				}
-				msg[i] = buf[i];
-			}
-			msg[i] = '\0';
-			err = got_error_msg(GOT_ERR_FETCH_FAILED, msg);
+			err = fetch_error(&buf[4], n - 4);
 			goto done;
 		}
 		err = parse_refline(&id_str, &refname, &server_capabilities,
@@ -584,20 +647,96 @@ fetch_pack(int fd, int packfd, struct got_object_id *packid,
 	 * will now send a "NAK" (meaning no common objects were found).
 	 */
 	if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
-		err = got_error(GOT_ERR_BAD_PACKET);
+		err = got_error_msg(GOT_ERR_BAD_PACKET,
+		    "unexpected message from server");
 		goto done;
 	}
 
 	if (chattygit)
 		fprintf(stderr, "fetching...\n");
+
+	if (my_capabilities != NULL &&
+	    strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
+		have_sidebands = 1;
+
 	packsz = 0;
 	while (1) {
-		ssize_t r, w;
-		err = readn(&r, fd, buf, sizeof buf);
-		if (err)
-			goto done;
-		if (r == 0)
-			break;
+		ssize_t r = 0, w;
+		int datalen = -1;
+
+		if (have_sidebands) {
+			err = read_pkthdr(&datalen, fd);
+			if (err)
+				goto done;
+			if (datalen <= 0)
+				break;
+
+			/* Read sideband channel ID (one byte). */
+			r = read(fd, buf, 1);
+			if (r == -1) {
+				err = got_error_from_errno("read");
+				goto done;
+			}
+			if (r != 1) {
+				err = got_error_msg(GOT_ERR_BAD_PACKET,
+				    "short packet");
+				goto done;
+			}
+			if (datalen > sizeof(buf) - 5) {
+				err = got_error_msg(GOT_ERR_BAD_PACKET,
+				    "bad packet length");
+				goto done;
+			}
+			datalen--; /* sideband ID has been read */
+			if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
+				/* Read packfile data. */
+				err = readn(&r, fd, buf, datalen);
+				if (err)
+					goto done;
+				if (r != datalen) {
+					err = got_error_msg(GOT_ERR_BAD_PACKET,
+					    "packet too short");
+					goto done;
+				}
+			} else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
+				err = readn(&r, fd, buf, datalen);
+				if (err)
+					goto done;
+				if (r != datalen) {
+					err = got_error_msg(GOT_ERR_BAD_PACKET,
+					    "packet too short");
+					goto done;
+				}
+				err = fetch_progress(ibuf, buf, r);
+				if (err)
+					goto done;
+				continue;
+			} else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
+				err = readn(&r, fd, buf, datalen);
+				if (err)
+					goto done;
+				if (r != datalen) {
+					err = got_error_msg(GOT_ERR_BAD_PACKET,
+					    "packet too short");
+					goto done;
+				}
+				err = fetch_error(buf, r);
+				goto done;
+			} else {
+				err = got_error_msg(GOT_ERR_BAD_PACKET,
+				    "unknown side-band received from server");
+				goto done;
+			}
+		} else {
+			/* No sideband channel. Every byte is packfile data. */
+			err = readn(&r, fd, buf, sizeof buf);
+			if (err)
+				goto done;
+			if (r <= 0)
+				break;
+		}
+
+		/* Write packfile data to temporary pack file. */
 		w = write(packfd, buf, r);
 		if (w == -1) {
 			err = got_error_from_errno("write");
@@ -607,7 +746,7 @@ fetch_pack(int fd, int packfd, struct got_object_id *packid,
 			err = got_error(GOT_ERR_IO);
 			goto done;
 		}
-		packsz += r;
+		packsz += w;
 	}
 	if (lseek(packfd, 0, SEEK_SET) == -1) {
 		err = got_error_from_errno("lseek");