Commit 6c41426143b93f57eebf1aaac35dcee4fa3fc282

Stefan Sperling 2021-03-30T01:26:42

cap pack file cache size at 1/8 of the current open file desciptor limit

diff --git a/lib/got_lib_repository.h b/lib/got_lib_repository.h
index 926534b..e26649f 100644
--- a/lib/got_lib_repository.h
+++ b/lib/got_lib_repository.h
@@ -28,8 +28,7 @@
 #define GOT_OBJECTS_PACK_DIR	"objects/pack"
 #define GOT_PACKED_REFS_FILE	"packed-refs"
 
-#define GOT_PACKIDX_CACHE_SIZE	64
-#define GOT_PACK_CACHE_SIZE	GOT_PACKIDX_CACHE_SIZE
+#define GOT_PACK_CACHE_SIZE	64
 
 struct got_repository {
 	char *path;
@@ -37,11 +36,17 @@ struct got_repository {
 	int gitdir_fd;
 
 	/* The pack index cache speeds up search for packed objects. */
-	struct got_packidx *packidx_cache[GOT_PACKIDX_CACHE_SIZE];
+	struct got_packidx *packidx_cache[GOT_PACK_CACHE_SIZE];
 
 	/* Open file handles for pack files. */
 	struct got_pack packs[GOT_PACK_CACHE_SIZE];
 
+	/*
+	 * The cache size limit may be lower than GOT_PACK_CACHE_SIZE,
+	 * depending on resource limits.
+	 */
+	int pack_cache_size;
+
 	/* Handles to child processes for reading loose objects. */
 	 struct got_privsep_child privsep_children[5];
 #define GOT_REPO_PRIVSEP_CHILD_OBJECT	0
diff --git a/lib/repository.c b/lib/repository.c
index 16a7584..56e526a 100644
--- a/lib/repository.c
+++ b/lib/repository.c
@@ -20,6 +20,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
+#include <sys/resource.h>
 
 #include <ctype.h>
 #include <endian.h>
@@ -615,9 +616,13 @@ got_repo_open(struct got_repository **repop, const char *path,
 	const struct got_error *err = NULL;
 	char *repo_path = NULL;
 	size_t i;
+	struct rlimit rl;
 
 	*repop = NULL;
 
+	if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
+		return got_error_from_errno("getrlimit");
+
 	repo = calloc(1, sizeof(*repo));
 	if (repo == NULL) {
 		err = got_error_from_errno("calloc");
@@ -647,6 +652,10 @@ got_repo_open(struct got_repository **repop, const char *path,
 	if (err)
 		goto done;
 
+	repo->pack_cache_size = GOT_PACK_CACHE_SIZE;
+	if (repo->pack_cache_size > rl.rlim_cur / 8)
+		repo->pack_cache_size = rl.rlim_cur / 8;
+
 	repo_path = realpath(path, NULL);
 	if (repo_path == NULL) {
 		err = got_error_from_errno2("realpath", path);
@@ -710,13 +719,13 @@ got_repo_close(struct got_repository *repo)
 	const struct got_error *err = NULL, *child_err;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		got_packidx_close(repo->packidx_cache[i]);
 	}
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packs[i].path_packfile == NULL)
 			break;
 		got_pack_close(&repo->packs[i]);
@@ -893,7 +902,7 @@ cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
 	const struct got_error *err = NULL;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		if (strcmp(repo->packidx_cache[i]->path_packidx,
@@ -901,7 +910,7 @@ cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
 		}
 	}
-	if (i == nitems(repo->packidx_cache)) {
+	if (i == repo->pack_cache_size) {
 		err = got_packidx_close(repo->packidx_cache[i - 1]);
 		if (err)
 			return err;
@@ -947,7 +956,7 @@ got_repo_search_packidx(struct got_packidx **packidx, int *idx,
 	int packdir_fd;
 
 	/* Search pack index cache. */
-	for (i = 0; i < nitems(repo->packidx_cache); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		if (repo->packidx_cache[i] == NULL)
 			break;
 		*idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
@@ -998,7 +1007,7 @@ got_repo_search_packidx(struct got_packidx **packidx, int *idx,
 			goto done;
 		}
 
-		for (i = 0; i < nitems(repo->packidx_cache); i++) {
+		for (i = 0; i < repo->pack_cache_size; i++) {
 			if (repo->packidx_cache[i] == NULL)
 				break;
 			if (strcmp(repo->packidx_cache[i]->path_packidx,
@@ -1094,7 +1103,7 @@ got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
 	if (packp)
 		*packp = NULL;
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		pack = &repo->packs[i];
 		if (pack->path_packfile == NULL)
 			break;
@@ -1102,7 +1111,7 @@ got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
 			return got_error(GOT_ERR_CACHE_DUP_ENTRY);
 	}
 
-	if (i == nitems(repo->packs)) {
+	if (i == repo->pack_cache_size) {
 		err = got_pack_close(&repo->packs[i - 1]);
 		if (err)
 			return err;
@@ -1159,7 +1168,7 @@ got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
 	struct got_pack *pack = NULL;
 	size_t i;
 
-	for (i = 0; i < nitems(repo->packs); i++) {
+	for (i = 0; i < repo->pack_cache_size; i++) {
 		pack = &repo->packs[i];
 		if (pack->path_packfile == NULL)
 			break;