Commit baa9fea074326a73792e66efbca9746a1bccc596

Stefan Sperling 2020-03-18T16:11:33

send indexing progress via an imsg

diff --git a/got/got.c b/got/got.c
index e4bbfb0..9fce345 100644
--- a/got/got.c
+++ b/got/got.c
@@ -970,7 +970,8 @@ done:
 }
 
 static const struct got_error *
-fetch_progress(void *arg, const char *message, off_t packfile_size)
+fetch_progress(void *arg, const char *message, off_t packfile_size,
+    int nobjects_total, int nobjects_indexed)
 {
 	int *did_something = arg;
 	char scaled[FMT_SCALED_STRSIZE];
@@ -978,8 +979,13 @@ fetch_progress(void *arg, const char *message, off_t packfile_size)
 	if (message) {
 		printf("\rserver: %s", message);
 		*did_something = 1;
-	} else if (packfile_size > 0 && fmt_scaled(packfile_size, scaled) == 0) {
-		printf("\rfetching... %*s", FMT_SCALED_STRSIZE, scaled);
+	} else if (packfile_size > 0 || nobjects_indexed > 0) {
+		printf("\rfetching...");
+		if (fmt_scaled(packfile_size, scaled) == 0)
+			printf(" %*s", FMT_SCALED_STRSIZE, scaled);
+		if (nobjects_indexed > 0)
+			printf(" indexed %d/%d objects", nobjects_indexed,
+			    nobjects_total);
 		*did_something = 1;
 	}
 	fflush(stdout);
diff --git a/include/got_fetch.h b/include/got_fetch.h
index 773e689..5c84e0c 100644
--- a/include/got_fetch.h
+++ b/include/got_fetch.h
@@ -40,7 +40,7 @@ const struct got_error *got_fetch_connect(int *, const char *, const char *,
 
 /* A callback function which gets invoked with progress information to print. */
 typedef const struct got_error *(*got_fetch_progress_cb)(void *,
-    const char *, off_t);
+    const char *, off_t, int, int);
 
 /*
  * Attempt to fetch a packfile from a server. This pack file will contain
diff --git a/lib/fetch.c b/lib/fetch.c
index cb01627..734df05 100644
--- a/lib/fetch.c
+++ b/lib/fetch.c
@@ -451,7 +451,7 @@ got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 			while ((s = strsep(&s0, "\r")) != NULL) {
 				if (*s == '\0')
 					continue;
-				err = progress_cb(progress_arg, s, 0);
+				err = progress_cb(progress_arg, s, 0, 0, 0);
 				if (err)
 					break;
 			}
@@ -460,7 +460,7 @@ got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 				goto done;
 		} else if (packfile_size_cur != packfile_size) {
 			err = progress_cb(progress_arg, NULL,
-			    packfile_size_cur);
+			    packfile_size_cur, 0, 0);
 			if (err)
 				break;
 			packfile_size = packfile_size_cur;
@@ -504,10 +504,22 @@ got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 	if (err != NULL)
 		goto done;
 	nidxfd = -1;
-	err = got_privsep_wait_index_pack_done(&idxibuf);
-	if (err != NULL)
-		goto done;
-	imsg_clear(&idxibuf);
+	done = 0;
+	while (!done) {
+		int nobjects_total, nobjects_indexed;
+		err = got_privsep_recv_index_progress(&done, &nobjects_total,
+		    &nobjects_indexed, &idxibuf);
+		if (err != NULL)
+			goto done;
+		if (nobjects_indexed != 0) {
+			err = progress_cb(progress_arg, NULL,
+			    packfile_size, nobjects_total,
+			    nobjects_indexed);
+			if (err)
+				break;
+		}
+		imsg_clear(&idxibuf);
+	}
 	if (close(imsg_idxfds[0]) == -1) {
 		err = got_error_from_errno("close");
 		goto done;
diff --git a/lib/got_lib_privsep.h b/lib/got_lib_privsep.h
index c557473..20e7744 100644
--- a/lib/got_lib_privsep.h
+++ b/lib/got_lib_privsep.h
@@ -116,6 +116,7 @@ enum got_imsg_type {
 	GOT_IMSG_FETCH_DOWNLOAD_PROGRESS,
 	GOT_IMSG_FETCH_DONE,
 	GOT_IMSG_IDXPACK_REQUEST,
+	GOT_IMSG_IDXPACK_PROGRESS,
 	GOT_IMSG_IDXPACK_DONE,
 
 	/* Messages related to pack files. */
@@ -276,6 +277,14 @@ struct got_imsg_fetch_download_progress {
 	off_t packfile_bytes;
 };
 
+/* Structure for GOT_IMSG_IDXPACK_PROGRESS data. */
+struct got_imsg_index_pack_progress {
+	/* Total number of objects in pack file. */
+	int nobjects_total;
+	/* Number of objects indexed so far. */
+	int nobjects_indexed;
+};
+
 /* Structure for GOT_IMSG_PACKIDX. */
 struct got_imsg_packidx {
 	size_t len;
@@ -356,8 +365,11 @@ const struct got_error *got_privsep_send_obj(struct imsgbuf *,
     struct got_object *);
 const struct got_error *got_privsep_send_index_pack_req(struct imsgbuf *, int,
     struct got_object_id *);
+const struct got_error *got_privsep_send_index_pack_progress(struct imsgbuf *,
+    int, int);
 const struct got_error *got_privsep_send_index_pack_done(struct imsgbuf *);
-const struct got_error *got_privsep_wait_index_pack_done(struct imsgbuf *);
+const struct got_error *got_privsep_recv_index_progress(int *, int *, int *,
+    struct imsgbuf *ibuf);
 const struct got_error *got_privsep_send_fetch_req(struct imsgbuf *, int,
     struct got_pathlist_head *);
 const struct got_error *got_privsep_send_fetch_symrefs(struct imsgbuf *,
diff --git a/lib/privsep.c b/lib/privsep.c
index 96c8759..986fccc 100644
--- a/lib/privsep.c
+++ b/lib/privsep.c
@@ -781,6 +781,22 @@ got_privsep_send_index_pack_req(struct imsgbuf *ibuf, int fd, struct got_object_
 }
 
 const struct got_error *
+got_privsep_send_index_pack_progress(struct imsgbuf *ibuf, int nobjects_total,
+    int nobjects_indexed)
+{
+	struct got_imsg_index_pack_progress iprogress;
+
+	iprogress.nobjects_total = nobjects_total;
+	iprogress.nobjects_indexed = nobjects_indexed;
+
+	if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_PROGRESS, 0, 0, -1,
+	    &iprogress, sizeof(iprogress)) == -1)
+		return got_error_from_errno("imsg_compose IDXPACK_PROGRESS");
+
+	return flush_imsg(ibuf);
+}
+
+const struct got_error *
 got_privsep_send_index_pack_done(struct imsgbuf *ibuf)
 {
 	if (imsg_compose(ibuf, GOT_IMSG_IDXPACK_DONE, 0, 0, -1, NULL, 0) == -1)
@@ -789,19 +805,54 @@ got_privsep_send_index_pack_done(struct imsgbuf *ibuf)
 }
 
 const struct got_error *
-got_privsep_wait_index_pack_done(struct imsgbuf *ibuf)
+got_privsep_recv_index_progress(int *done, int *nobjects_total,
+    int *nobjects_indexed, struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	struct imsg imsg;
+	struct got_imsg_index_pack_progress *iprogress;
+	size_t datalen;
+
+	*done = 0;
+	*nobjects_total = 0;
+	*nobjects_indexed = 0;
 
 	err = got_privsep_recv_imsg(&imsg, ibuf, 0);
 	if (err)
 		return err;
-	if (imsg.hdr.type == GOT_IMSG_IDXPACK_DONE)
-		return NULL;
-	else
-		return got_error(GOT_ERR_PRIVSEP_MSG);
+
+	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
+	switch (imsg.hdr.type) {
+	case GOT_IMSG_ERROR:
+		if (datalen < sizeof(struct got_imsg_error)) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		err = recv_imsg_error(&imsg, datalen);
+		break;
+	case GOT_IMSG_IDXPACK_PROGRESS:
+		if (datalen < sizeof(*iprogress)) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		iprogress = (struct got_imsg_index_pack_progress *)imsg.data;
+		*nobjects_total = iprogress->nobjects_total;
+		*nobjects_indexed = iprogress->nobjects_indexed;
+		break;
+	case GOT_IMSG_IDXPACK_DONE:
+		if (datalen != 0) {
+			err = got_error(GOT_ERR_PRIVSEP_LEN);
+			break;
+		}
+		*done = 1;
+		break;
+	default:
+		err = got_error(GOT_ERR_PRIVSEP_MSG);
+		break;
+	}
+
 	imsg_free(&imsg);
+	return err;
 }
 
 const struct got_error *
diff --git a/libexec/got-index-pack/got-index-pack.c b/libexec/got-index-pack/got-index-pack.c
index 1e04463..4247c3e 100644
--- a/libexec/got-index-pack/got-index-pack.c
+++ b/libexec/got-index-pack/got-index-pack.c
@@ -1085,7 +1085,8 @@ objectcrc(FILE *f, Object *o)
 }
 
 int
-indexpack(int packfd, int idxfd, struct got_object_id *packhash)
+indexpack(int packfd, int idxfd, struct got_object_id *packhash,
+    struct imsgbuf *ibuf)
 {
 	char hdr[4*3], buf[8];
 	int nobj, nvalid, nbig, n, i, step;
@@ -1117,15 +1118,13 @@ indexpack(int packfd, int idxfd, struct got_object_id *packhash)
 	if(!step)
 		step++;
 	while (nvalid != nobj) {
-		fprintf(stderr, "indexing (%d/%d):", nvalid, nobj);
+		got_privsep_send_index_pack_progress(ibuf, nobj, nvalid);
 		n = 0;
 		for (i = 0; i < nobj; i++) {
 			if (valid[i]) {
 				n++;
 				continue;
 			}
-			if (i % step == 0)
-				fprintf(stderr, ".");
 			if (!objects[i]) {
 				o = emalloc(sizeof(Object));
 				o->off = ftello(f);
@@ -1144,7 +1143,6 @@ indexpack(int packfd, int idxfd, struct got_object_id *packhash)
 			if(objectcrc(f, o) == -1)
 				return -1;
 		}
-		fprintf(stderr, "\n");
 		if (n == nvalid) {
 			errx(1, "fix point reached too early: %d/%d", nvalid, nobj);
 			goto error;
@@ -1254,7 +1252,7 @@ main(int argc, char **argv)
 	}
 	idxfd = imsg.fd;
 
-	indexpack(packfd, idxfd, &packhash);
+	indexpack(packfd, idxfd, &packhash, &ibuf);
 done:
 	if(err != NULL)
 		got_privsep_send_error(&ibuf, err);