Commit 710f3f4e5168164687c5586641920d82960b0a3a

Stefan Sperling 2018-11-05T15:26:18

make got-read-pack pre-seed the main process mini commit cache

diff --git a/lib/got_lib_object_parse.h b/lib/got_lib_object_parse.h
index 0be7fad..45ef92b 100644
--- a/lib/got_lib_object_parse.h
+++ b/lib/got_lib_object_parse.h
@@ -52,6 +52,7 @@ const struct got_error *got_object_packed_read_privsep(struct got_object **,
 const struct got_error *got_object_read_packed_commit_privsep(
     struct got_commit_object **, struct got_object *, struct got_pack *);
 const struct got_error *got_object_read_packed_mini_commit_privsep(
-    struct got_mini_commit_object **, struct got_object *, struct got_pack *);
+    struct got_mini_commit_object **, struct got_object *, struct got_pack *,
+    struct got_repository *);
 const struct got_error *got_object_read_packed_tree_privsep(
     struct got_tree_object **, struct got_object *, struct got_pack *);
diff --git a/lib/got_lib_privsep.h b/lib/got_lib_privsep.h
index bd18a5b..4a84a70 100644
--- a/lib/got_lib_privsep.h
+++ b/lib/got_lib_privsep.h
@@ -150,6 +150,13 @@ struct got_imsg_commit_object_mini {
 	struct tm tm_committer;
 	int nparents;
 
+	/*
+	 * Set if this commit is a parent of the requested commit.
+	 * Allows got-read-pack to seed the main process mini-commit cache.
+	 */
+	int is_parent;
+	uint8_t parent_id[SHA1_DIGEST_LENGTH];
+
 	/* Followed by 'nparents' SHA1_DIGEST_LENGTH length strings */
 } __attribute__((__packed__));
 
@@ -213,11 +220,12 @@ const struct got_error *got_privsep_recv_obj(struct got_object **,
 const struct got_error *got_privsep_send_commit(struct imsgbuf *,
     struct got_commit_object *);
 const struct got_error *got_privsep_send_mini_commit(struct imsgbuf *,
-    struct got_mini_commit_object *);
+    struct got_mini_commit_object *, struct got_object_id *);
 const struct got_error *got_privsep_recv_commit(struct got_commit_object **,
     struct imsgbuf *);
 const struct got_error *got_privsep_recv_mini_commit(
-    struct got_mini_commit_object **, struct imsgbuf *);
+    struct got_mini_commit_object **, struct got_object_id **,
+    struct imsgbuf *);
 const struct got_error *got_privsep_recv_tree(struct got_tree_object **,
     struct imsgbuf *);
 const struct got_error *got_privsep_send_tree(struct imsgbuf *,
diff --git a/lib/object.c b/lib/object.c
index fff88ff..959246b 100644
--- a/lib/object.c
+++ b/lib/object.c
@@ -378,7 +378,7 @@ open_mini_commit(struct got_mini_commit_object **commit,
 				return err;
 		}
 		err = got_object_read_packed_mini_commit_privsep(commit, obj,
-		    pack);
+		    pack, repo);
 	} else {
 		int fd;
 		err = open_loose_object(&fd, obj, repo);
@@ -1179,12 +1179,11 @@ request_mini_commit(struct got_mini_commit_object **commit,
 	struct imsgbuf *ibuf;
 
 	ibuf = repo->privsep_children[GOT_REPO_PRIVSEP_CHILD_COMMIT].ibuf;
-
 	err = got_privsep_send_mini_commit_req(ibuf, fd, obj);
 	if (err)
 		return err;
 
-	return got_privsep_recv_mini_commit(commit, ibuf);
+	return got_privsep_recv_mini_commit(commit, NULL, ibuf);
 }
 
 const struct got_error *
@@ -1201,17 +1200,32 @@ got_object_read_packed_commit_privsep(struct got_commit_object **commit,
 }
 
 const struct got_error *
-got_object_read_packed_mini_commit_privsep(struct got_mini_commit_object **commit,
-    struct got_object *obj, struct got_pack *pack)
+got_object_read_packed_mini_commit_privsep(
+    struct got_mini_commit_object **commit, struct got_object *obj,
+    struct got_pack *pack, struct got_repository *repo)
 {
 	const struct got_error *err = NULL;
+	struct got_object_id *pid = NULL;
 
 	err = got_privsep_send_mini_commit_req(pack->privsep_child->ibuf, -1,
 	    obj);
 	if (err)
 		return err;
 
-	return got_privsep_recv_mini_commit(commit, pack->privsep_child->ibuf);
+	 while (1) {
+		err = got_privsep_recv_mini_commit(commit, &pid,
+		    pack->privsep_child->ibuf);
+		if (err || pid == NULL)
+			break;
+
+		/* got-read-pack has sent a parent commit; cache it. */
+		err = got_repo_cache_mini_commit(repo, pid, *commit);
+		free(pid);
+		if (err)
+			break;
+	}
+
+	return err;
 }
 
 const struct got_error *
diff --git a/lib/object_cache.c b/lib/object_cache.c
index 6d49541..112f328 100644
--- a/lib/object_cache.c
+++ b/lib/object_cache.c
@@ -35,7 +35,7 @@
 #define GOT_OBJECT_CACHE_SIZE_OBJ		1024
 #define GOT_OBJECT_CACHE_SIZE_TREE		2048
 #define GOT_OBJECT_CACHE_SIZE_COMMIT		512
-#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT	512
+#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT	32
 
 const struct got_error *
 got_object_cache_init(struct got_object_cache *cache,
diff --git a/lib/privsep.c b/lib/privsep.c
index d5e0f72..ff2cfa6 100644
--- a/lib/privsep.c
+++ b/lib/privsep.c
@@ -39,6 +39,7 @@
 #include "got_lib_inflate.h"
 #include "got_lib_object.h"
 #include "got_lib_object_parse.h"
+#include "got_lib_object_cache.h"
 #include "got_lib_privsep.h"
 #include "got_lib_pack.h"
 
@@ -488,7 +489,7 @@ done:
 
 const struct got_error *
 got_privsep_send_mini_commit(struct imsgbuf *ibuf,
-    struct got_mini_commit_object *commit)
+    struct got_mini_commit_object *commit, struct got_object_id *parent_id)
 {
 	const struct got_error *err = NULL;
 	struct got_imsg_commit_object_mini icommit;
@@ -501,6 +502,13 @@ got_privsep_send_mini_commit(struct imsgbuf *ibuf,
 	    sizeof(icommit.tm_committer));
 	icommit.nparents = commit->nparents;
 
+	icommit.is_parent = parent_id ? 1 : 0;
+	if (parent_id)
+		memcpy(icommit.parent_id, parent_id->sha1,
+		    sizeof(icommit.parent_id));
+	else
+		memset(&icommit.parent_id, 0, sizeof(icommit.parent_id));
+
 	total = sizeof(icommit) + icommit.nparents * SHA1_DIGEST_LENGTH;
 
 	buf = malloc(total);
@@ -523,7 +531,8 @@ got_privsep_send_mini_commit(struct imsgbuf *ibuf,
 	free(buf);
 	if (err)
 		return err;
-	return flush_imsg(ibuf);
+
+	return parent_id ? NULL : flush_imsg(ibuf);
 }
 
 const struct got_error *
@@ -680,15 +689,47 @@ got_privsep_recv_commit(struct got_commit_object **commit, struct imsgbuf *ibuf)
 	return err;
 }
 
+static const struct got_error *
+get_mini_commit(struct got_mini_commit_object **commit,
+    struct got_imsg_commit_object_mini *icommit, uint8_t *data)
+{
+	const struct got_error *err = NULL;
+	int i;
+
+	*commit = got_object_mini_commit_alloc_partial();
+	if (*commit == NULL)
+		return got_error_from_errno();
+
+	memcpy((*commit)->tree_id->sha1, icommit->tree_id,
+	    SHA1_DIGEST_LENGTH);
+	memcpy(&(*commit)->tm_committer, &icommit->tm_committer,
+	    sizeof((*commit)->tm_committer));
+
+	for (i = 0; i < icommit->nparents; i++) {
+		struct got_object_qid *qid;
+
+		err = got_object_qid_alloc_partial(&qid);
+		if (err)
+			break;
+
+		memcpy(qid->id,
+		    data + sizeof(*icommit) + (i * SHA1_DIGEST_LENGTH),
+		    sizeof(*qid->id));
+		SIMPLEQ_INSERT_TAIL(&(*commit)->parent_ids, qid, entry);
+		(*commit)->nparents++;
+	}
+
+	return err;
+}
+
 const struct got_error *
 got_privsep_recv_mini_commit(struct got_mini_commit_object **commit,
-    struct imsgbuf *ibuf)
+    struct got_object_id **parent_id, struct imsgbuf *ibuf)
 {
 	const struct got_error *err = NULL;
 	struct imsg imsg;
 	struct got_imsg_commit_object_mini icommit;
-	size_t len, datalen;
-	int i;
+	size_t datalen;
 	const size_t min_datalen =
 	    MIN(sizeof(struct got_imsg_error),
 	    sizeof(struct got_imsg_commit_object_mini));
@@ -702,7 +743,6 @@ got_privsep_recv_mini_commit(struct got_mini_commit_object **commit,
 
 	data = imsg.data;
 	datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
-	len = 0;
 
 	switch (imsg.hdr.type) {
 	case GOT_IMSG_ERROR:
@@ -724,31 +764,18 @@ got_privsep_recv_mini_commit(struct got_mini_commit_object **commit,
 			err = got_error(GOT_ERR_PRIVSEP_LEN);
 			break;
 		}
-		len += sizeof(icommit);
-
-		*commit = got_object_mini_commit_alloc_partial();
-		if (*commit == NULL) {
-			err = got_error_from_errno();
-			break;
-		}
-
-		memcpy((*commit)->tree_id->sha1, icommit.tree_id,
-		    SHA1_DIGEST_LENGTH);
-		memcpy(&(*commit)->tm_committer, &icommit.tm_committer,
-		    sizeof((*commit)->tm_committer));
-
-		for (i = 0; i < icommit.nparents; i++) {
-			struct got_object_qid *qid;
-
-			err = got_object_qid_alloc_partial(&qid);
-			if (err)
+		/* got-read-pack might send us parent commits for caching */
+		if (icommit.is_parent) {
+			if (parent_id == NULL) {
+				err = got_error(GOT_ERR_PRIVSEP_MSG);
 				break;
-
-			memcpy(qid->id, data + len + i * SHA1_DIGEST_LENGTH,
-			    sizeof(*qid->id));
-			SIMPLEQ_INSERT_TAIL(&(*commit)->parent_ids, qid, entry);
-			(*commit)->nparents++;
-		}
+			}
+			*parent_id = malloc(sizeof(**parent_id));
+			memcpy((*parent_id)->sha1, &icommit.parent_id,
+			    sizeof(*(*parent_id)->sha1));
+		} else if (parent_id)
+			*parent_id = NULL;
+		err = get_mini_commit(commit, &icommit, data);
 		break;
 	default:
 		err = got_error(GOT_ERR_PRIVSEP_MSG);
diff --git a/libexec/got-read-commit/got-read-commit.c b/libexec/got-read-commit/got-read-commit.c
index b662f7b..8ae1f39 100644
--- a/libexec/got-read-commit/got-read-commit.c
+++ b/libexec/got-read-commit/got-read-commit.c
@@ -181,7 +181,7 @@ main(int argc, char *argv[])
 			err = read_commit_object_mini(&commit, obj, f);
 			if (err)
 				goto done;
-			err = got_privsep_send_mini_commit(&ibuf, commit);
+			err = got_privsep_send_mini_commit(&ibuf, commit, NULL);
 			got_object_mini_commit_close(commit);
 		} else {
 			struct got_commit_object *commit;
diff --git a/libexec/got-read-pack/got-read-pack.c b/libexec/got-read-pack/got-read-pack.c
index f9e2970..f1be1b6 100644
--- a/libexec/got-read-pack/got-read-pack.c
+++ b/libexec/got-read-pack/got-read-pack.c
@@ -145,6 +145,60 @@ commit_request(struct imsg *imsg, struct imsgbuf *ibuf, struct got_pack *pack,
 }
 
 static const struct got_error *
+send_parent_commit(size_t *totlen, struct imsgbuf *ibuf,
+    struct got_object_id *pid, struct got_pack *pack,
+    struct got_packidx *packidx, int recurse)
+{
+	const struct got_error *err = NULL;
+	struct got_object *pobj;
+	struct got_mini_commit_object *pcommit = NULL;
+	uint8_t *buf;
+	size_t len, plen;
+	int idx;
+
+	idx = got_packidx_get_object_idx(packidx, pid);
+	if (idx == -1) /* parent commit not in same pack file */
+		return NULL;
+
+	err = got_packfile_open_object(&pobj, pack, packidx, idx, pid);
+	if (err)
+		return err;
+
+	err = got_packfile_extract_object_to_mem(&buf, &len, pobj,
+	    pack);
+	if (err) {
+		got_object_close(pobj);
+		return err;
+	}
+
+	pobj->size = len;
+	err = got_object_parse_mini_commit(&pcommit, buf, len);
+	free(buf);
+
+	plen = sizeof(*pcommit) + (pcommit->nparents * SHA1_DIGEST_LENGTH);
+	if (*totlen + plen >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+		goto done;
+	*totlen += plen;
+
+	err = got_privsep_send_mini_commit(ibuf, pcommit, pid);
+	if (err)
+		goto done;
+
+	/* Send the first grandparent along as well if there is room. */
+	if (recurse > 0) {
+		struct got_object_qid *qid;
+		qid = SIMPLEQ_FIRST(&pcommit->parent_ids);
+		if (qid)
+			err = send_parent_commit(totlen, ibuf, qid->id, pack,
+			    packidx, recurse - 1);
+	}
+done:
+	got_object_close(pobj);
+	got_object_mini_commit_close(pcommit);
+	return err;
+}
+
+static const struct got_error *
 mini_commit_request(struct imsg *imsg, struct imsgbuf *ibuf,
     struct got_pack *pack, struct got_packidx *packidx,
     struct got_object_cache *objcache)
@@ -153,7 +207,8 @@ mini_commit_request(struct imsg *imsg, struct imsgbuf *ibuf,
 	struct got_object *obj = NULL;
 	struct got_mini_commit_object *commit = NULL;
 	uint8_t *buf;
-	size_t len;
+	size_t len, totlen;
+	struct got_object_qid *qid;
 
 	err = get_object(&obj, imsg, ibuf, pack, packidx, objcache,
 	    GOT_OBJ_TYPE_COMMIT);
@@ -168,9 +223,25 @@ mini_commit_request(struct imsg *imsg, struct imsgbuf *ibuf,
 	err = got_object_parse_mini_commit(&commit, buf, len);
 	free(buf);
 
-	err = got_privsep_send_mini_commit(ibuf, commit);
-	if (obj)
-		got_object_close(obj);
+	/*
+	 * Try to pre-seed the main process mini-commit cache with parent
+	 * commits from this pack file. This makes more efficient use of
+	 * imsg pipe buffers per system call.
+	 */
+	totlen = sizeof(struct got_imsg_commit_object_mini) +
+	    (commit->nparents * sizeof(SHA1_DIGEST_LENGTH));
+	SIMPLEQ_FOREACH(qid, &commit->parent_ids, entry) {
+		if (totlen >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
+			break;
+		err = send_parent_commit(&totlen, ibuf, qid->id,
+		    pack, packidx, 1);
+		if (err)
+			goto done;
+	}
+
+	err = got_privsep_send_mini_commit(ibuf, commit, NULL);
+done:
+	got_object_close(obj);
 	got_object_mini_commit_close(commit);
 	if (err) {
 		if (err->code == GOT_ERR_PRIVSEP_PIPE)