Commit e32baab7e125247f6fe514984040708eb0fc1d0c

Stefan Sperling 2018-11-05T12:39:49

add an object cache for mini commits

diff --git a/lib/commit_graph.c b/lib/commit_graph.c
index be5add9..08c1ddd 100644
--- a/lib/commit_graph.c
+++ b/lib/commit_graph.c
@@ -194,7 +194,7 @@ detect_changed_path(int *changed, struct got_commit_object_mini *commit,
 			*changed = 1; /* The path was created in this commit. */
 		free(obj_id);
 	} else {
-		err = got_object_open_mini_commit(&pcommit, repo, pid->id);
+		err = got_object_open_as_mini_commit(&pcommit, repo, pid->id);
 		if (err)
 			goto done;
 
@@ -460,7 +460,7 @@ got_commit_graph_open(struct got_commit_graph **graph,
 
 	*graph = NULL;
 
-	err = got_object_open_mini_commit(&commit, repo, commit_id);
+	err = got_object_open_as_mini_commit(&commit, repo, commit_id);
 	if (err)
 		return err;
 
@@ -553,7 +553,7 @@ fetch_commits_from_open_branches(int *ncommits,
 		commit_id = &graph->tips[i].id;
 		child_node = graph->tips[i].node;
 
-		err = got_object_open_mini_commit(&commit, repo, commit_id);
+		err = got_object_open_as_mini_commit(&commit, repo, commit_id);
 		if (err)
 			break;
 
@@ -635,7 +635,7 @@ got_commit_graph_iter_start(struct got_commit_graph *graph,
 	if (start_node == NULL)
 		return got_error(GOT_ERR_NO_OBJ);
 
-	err = got_object_open_mini_commit(&commit, repo, &start_node->id);
+	err = got_object_open_as_mini_commit(&commit, repo, &start_node->id);
 	if (err)
 		return err;
 
diff --git a/lib/got_lib_object.h b/lib/got_lib_object.h
index 95f019e..5eccdfb 100644
--- a/lib/got_lib_object.h
+++ b/lib/got_lib_object.h
@@ -55,9 +55,13 @@ struct got_commit_object_mini {
 	unsigned int nparents;
 	struct got_object_id_queue parent_ids;
 	struct tm tm_committer;	/* UTC */
+	int refcnt;		/* > 0 if open and/or cached */
 };
 
 const struct got_error *
-got_object_open_mini_commit(struct got_commit_object_mini **,
-    struct got_repository *, struct got_object_id *);
+got_object_mini_commit_open(struct got_commit_object_mini **,
+    struct got_repository *, struct got_object *);
 void got_object_mini_commit_close(struct got_commit_object_mini *);
+const struct got_error *got_object_open_as_mini_commit(
+    struct got_commit_object_mini **, struct got_repository *,
+    struct got_object_id *);
diff --git a/lib/got_lib_object_cache.h b/lib/got_lib_object_cache.h
index d574268..424cbaa 100644
--- a/lib/got_lib_object_cache.h
+++ b/lib/got_lib_object_cache.h
@@ -18,6 +18,7 @@ enum got_object_cache_type {
 	GOT_OBJECT_CACHE_TYPE_OBJ,
 	GOT_OBJECT_CACHE_TYPE_TREE,
 	GOT_OBJECT_CACHE_TYPE_COMMIT,
+	GOT_OBJECT_CACHE_TYPE_MINI_COMMIT,
 };
 
 struct got_object_cache_entry {
@@ -26,6 +27,7 @@ struct got_object_cache_entry {
 		struct got_object *obj;
 		struct got_tree_object *tree;
 		struct got_commit_object *commit;
+		struct got_commit_object_mini *mini_commit;
 	} data;
 };
 
diff --git a/lib/got_lib_repository.h b/lib/got_lib_repository.h
index feabca1..3eb7a0d 100644
--- a/lib/got_lib_repository.h
+++ b/lib/got_lib_repository.h
@@ -38,6 +38,7 @@ struct got_repository {
 	struct got_object_cache objcache;
 	struct got_object_cache treecache;
 	struct got_object_cache commitcache;
+	struct got_object_cache minicommitcache;
 };
 
 const struct got_error*got_repo_cache_object(struct got_repository *,
@@ -52,6 +53,10 @@ const struct got_error*got_repo_cache_commit(struct got_repository *,
     struct got_object_id *, struct got_commit_object *);
 struct got_commit_object *got_repo_get_cached_commit(struct got_repository *,
     struct got_object_id *);
+const struct got_error*got_repo_cache_mini_commit(struct got_repository *,
+    struct got_object_id *, struct got_commit_object_mini *);
+struct got_commit_object_mini *got_repo_get_cached_mini_commit(
+    struct got_repository *, struct got_object_id *);
 const struct got_error *got_repo_cache_packidx(struct got_repository *,
     struct got_packidx *);
 const struct got_error *got_repo_search_packidx(struct got_packidx **, int *,
diff --git a/lib/object.c b/lib/object.c
index de22dce..cdf4028 100644
--- a/lib/object.c
+++ b/lib/object.c
@@ -316,13 +316,54 @@ open_commit(struct got_commit_object **commit,
 	return err;
 }
 
+const struct got_error *
+got_object_open_as_commit(struct got_commit_object **commit,
+    struct got_repository *repo, struct got_object_id *id)
+{
+	const struct got_error *err;
+	struct got_object *obj;
+
+	*commit = got_repo_get_cached_commit(repo, id);
+	if (*commit != NULL) {
+		(*commit)->refcnt++;
+		return NULL;
+	}
+
+	err = got_object_open(&obj, repo, id);
+	if (err)
+		return err;
+	if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) {
+		err = got_error(GOT_ERR_OBJ_TYPE);
+		goto done;
+	}
+
+	err = open_commit(commit, repo, obj, 0);
+done:
+	got_object_close(obj);
+	return err;
+}
+
+const struct got_error *
+got_object_commit_open(struct got_commit_object **commit,
+    struct got_repository *repo, struct got_object *obj)
+{
+	return open_commit(commit, repo, obj, 1);
+}
+
 static const struct got_error *
 open_mini_commit(struct got_commit_object_mini **commit,
-    struct got_repository *repo, struct got_object *obj)
+    struct got_repository *repo, struct got_object *obj, int check_cache)
 {
 	const struct got_error *err = NULL;
 
-	*commit = NULL;
+	if (check_cache) {
+		*commit = got_repo_get_cached_mini_commit(repo, &obj->id);
+		if (*commit != NULL) {
+			(*commit)->refcnt++;
+			return NULL;
+		}
+	} else
+		*commit = NULL;
 
 	if (obj->type != GOT_OBJ_TYPE_COMMIT)
 		return got_error(GOT_ERR_OBJ_TYPE);
@@ -348,17 +389,22 @@ open_mini_commit(struct got_commit_object_mini **commit,
 		close(fd);
 	}
 
+	if (err == NULL) {
+		(*commit)->refcnt++;
+		err = got_repo_cache_mini_commit(repo, &obj->id, *commit);
+	}
+
 	return err;
 }
 
 const struct got_error *
-got_object_open_as_commit(struct got_commit_object **commit,
+got_object_open_as_mini_commit(struct got_commit_object_mini **commit,
     struct got_repository *repo, struct got_object_id *id)
 {
 	const struct got_error *err;
 	struct got_object *obj;
 
-	*commit = got_repo_get_cached_commit(repo, id);
+	*commit = got_repo_get_cached_mini_commit(repo, id);
 	if (*commit != NULL) {
 		(*commit)->refcnt++;
 		return NULL;
@@ -372,38 +418,17 @@ got_object_open_as_commit(struct got_commit_object **commit,
 		goto done;
 	}
 
-	err = open_commit(commit, repo, obj, 0);
+	err = open_mini_commit(commit, repo, obj, 0);
 done:
 	got_object_close(obj);
 	return err;
 }
 
 const struct got_error *
-got_object_commit_open(struct got_commit_object **commit,
+got_object_mini_commit_open(struct got_commit_object_mini **commit,
     struct got_repository *repo, struct got_object *obj)
 {
-	return open_commit(commit, repo, obj, 1);
-}
-
-const struct got_error *
-got_object_open_mini_commit(struct got_commit_object_mini **commit,
-    struct got_repository *repo, struct got_object_id *id)
-{
-	const struct got_error *err;
-	struct got_object *obj;
-
-	err = got_object_open(&obj, repo, id);
-	if (err)
-		return err;
-	if (got_object_get_type(obj) != GOT_OBJ_TYPE_COMMIT) {
-		err = got_error(GOT_ERR_OBJ_TYPE);
-		goto done;
-	}
-
-	err = open_mini_commit(commit, repo, obj);
-done:
-	got_object_close(obj);
-	return err;
+	return open_mini_commit(commit, repo, obj, 1);
 }
 
 const struct got_error *
diff --git a/lib/object_cache.c b/lib/object_cache.c
index 24b34b5..883a84a 100644
--- a/lib/object_cache.c
+++ b/lib/object_cache.c
@@ -32,9 +32,10 @@
 #include "got_lib_object_idcache.h"
 #include "got_lib_object_cache.h"
 
-#define GOT_OBJECT_CACHE_SIZE_OBJ	1024
-#define GOT_OBJECT_CACHE_SIZE_TREE	2048
-#define GOT_OBJECT_CACHE_SIZE_COMMIT	512
+#define GOT_OBJECT_CACHE_SIZE_OBJ		1024
+#define GOT_OBJECT_CACHE_SIZE_TREE		2048
+#define GOT_OBJECT_CACHE_SIZE_COMMIT		512
+#define GOT_OBJECT_CACHE_SIZE_MINI_COMMIT	512
 
 const struct got_error *
 got_object_cache_init(struct got_object_cache *cache,
@@ -52,6 +53,9 @@ got_object_cache_init(struct got_object_cache *cache,
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		size = GOT_OBJECT_CACHE_SIZE_COMMIT;
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		size = GOT_OBJECT_CACHE_SIZE_MINI_COMMIT;
+		break;
 	}
 
 	cache->idcache = got_object_idcache_alloc(size);
@@ -85,6 +89,9 @@ got_object_cache_add(struct got_object_cache *cache, struct got_object_id *id, v
 		case GOT_OBJECT_CACHE_TYPE_COMMIT:
 			got_object_commit_close(ce->data.commit);
 			break;
+		case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+			got_object_mini_commit_close(ce->data.mini_commit);
+			break;
 		}
 		free(ce);
 		cache->cache_evict++;
@@ -104,6 +111,9 @@ got_object_cache_add(struct got_object_cache *cache, struct got_object_id *id, v
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		ce->data.commit = (struct got_commit_object *)item;
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		ce->data.mini_commit = (struct got_commit_object_mini *)item;
+		break;
 	}
 
 	err = got_object_idcache_add(cache->idcache, id, ce);
@@ -132,6 +142,8 @@ got_object_cache_get(struct got_object_cache *cache, struct got_object_id *id)
 			return ce->data.tree;
 		case GOT_OBJECT_CACHE_TYPE_COMMIT:
 			return ce->data.commit;
+		case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+			return ce->data.mini_commit;
 		}
 	}
 
@@ -157,6 +169,7 @@ void check_refcount(struct got_object_id *id, void *data, void *arg)
 	struct got_object *obj;
 	struct got_tree_object *tree;
 	struct got_commit_object *commit;
+	struct got_commit_object_mini *mini_commit;
 	char *id_str;
 
 	if (got_object_id_str(&id_str, id) != NULL)
@@ -184,6 +197,13 @@ void check_refcount(struct got_object_id *id, void *data, void *arg)
 		fprintf(stderr, "commit %s has %d unclaimed references\n",
 		    id_str, commit->refcnt - 1);
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		mini_commit = ce->data.mini_commit;
+		if (mini_commit->refcnt == 1)
+			break;
+		fprintf(stderr, "commit %s has %d unclaimed references\n",
+		    id_str, mini_commit->refcnt - 1);
+		break;
 	}
 	free(id_str);
 }
@@ -203,6 +223,9 @@ got_object_cache_close(struct got_object_cache *cache)
 	case GOT_OBJECT_CACHE_TYPE_COMMIT:
 		print_cache_stats(cache, "commit");
 		break;
+	case GOT_OBJECT_CACHE_TYPE_MINI_COMMIT:
+		print_cache_stats(cache, "mini-commit");
+		break;
 	}
 
 	got_object_idcache_for_each(cache->idcache, check_refcount, cache);
diff --git a/lib/object_parse.c b/lib/object_parse.c
index ebf4613..ce15a31 100644
--- a/lib/object_parse.c
+++ b/lib/object_parse.c
@@ -314,6 +314,12 @@ got_object_mini_commit_close(struct got_commit_object_mini *commit)
 {
 	struct got_object_qid *qid;
 
+	if (commit->refcnt > 0) {
+		commit->refcnt--;
+		if (commit->refcnt > 0)
+			return;
+	}
+
 	while (!SIMPLEQ_EMPTY(&commit->parent_ids)) {
 		qid = SIMPLEQ_FIRST(&commit->parent_ids);
 		SIMPLEQ_REMOVE_HEAD(&commit->parent_ids, entry);
diff --git a/lib/repository.c b/lib/repository.c
index d82e38e..751a7ce 100644
--- a/lib/repository.c
+++ b/lib/repository.c
@@ -232,6 +232,28 @@ got_repo_get_cached_commit(struct got_repository *repo,
 }
 
 const struct got_error *
+got_repo_cache_mini_commit(struct got_repository *repo,
+    struct got_object_id *id, struct got_commit_object_mini *commit)
+{
+#ifndef GOT_NO_OBJ_CACHE
+	const struct got_error *err = NULL;
+	err = got_object_cache_add(&repo->minicommitcache, id, commit);
+	if (err)
+		return err;
+	commit->refcnt++;
+#endif
+	return NULL;
+}
+
+struct got_commit_object_mini *
+got_repo_get_cached_mini_commit(struct got_repository *repo,
+    struct got_object_id *id)
+{
+	return (struct got_commit_object_mini *)got_object_cache_get(
+	    &repo->minicommitcache, id);
+}
+
+const struct got_error *
 open_repo(struct got_repository *repo, const char *path)
 {
 	const struct got_error *err = NULL;
@@ -354,6 +376,10 @@ got_repo_open(struct got_repository **repop, const char *path)
 	    GOT_OBJECT_CACHE_TYPE_COMMIT);
 	if (err)
 		goto done;
+	err = got_object_cache_init(&repo->minicommitcache,
+	    GOT_OBJECT_CACHE_TYPE_MINI_COMMIT);
+	if (err)
+		goto done;
 
 	normpath = got_path_normalize(abspath);
 	if (normpath == NULL) {
@@ -413,6 +439,7 @@ got_repo_close(struct got_repository *repo)
 	got_object_cache_close(&repo->objcache);
 	got_object_cache_close(&repo->treecache);
 	got_object_cache_close(&repo->commitcache);
+	got_object_cache_close(&repo->minicommitcache);
 
 	for (i = 0; i < nitems(repo->privsep_children); i++) {
 		if (repo->privsep_children[i].imsg_fd == -1)