Commit e93fb944fe7b32d96fafd4726043a28062563b54
2022-05-10T11:34:16
map delta cache file into memory if possible while writing a pack file with a fix from + ok op@
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
diff --git a/lib/pack_create.c b/lib/pack_create.c
index 65a0185..71ea8bf 100644
--- a/lib/pack_create.c
+++ b/lib/pack_create.c
@@ -21,8 +21,10 @@
#include <sys/uio.h>
#include <sys/stat.h>
#include <sys/time.h>
+#include <sys/mman.h>
#include <endian.h>
+#include <errno.h>
#include <stdint.h>
#include <imsg.h>
#include <stdio.h>
@@ -1554,6 +1556,23 @@ hcopy(FILE *fsrc, FILE *fdst, off_t len, SHA1_CTX *ctx)
return NULL;
}
+const struct got_error *
+hcopy_mmap(uint8_t *src, off_t src_offset, size_t src_size,
+ FILE *fdst, off_t len, SHA1_CTX *ctx)
+{
+ size_t n;
+
+ if (src_offset + len > src_size)
+ return got_error(GOT_ERR_RANGE);
+
+ SHA1Update(ctx, src + src_offset, len);
+ n = fwrite(src + src_offset, 1, len, fdst);
+ if (n != len)
+ return got_ferror(fdst, GOT_ERR_IO);
+
+ return NULL;
+}
+
static void
putbe32(char *b, uint32_t n)
{
@@ -1683,8 +1702,9 @@ deltahdr(off_t *packfile_size, SHA1_CTX *ctx, FILE *packfile,
static const struct got_error *
write_packed_object(off_t *packfile_size, FILE *packfile,
- FILE *delta_cache, struct got_pack_meta *m, int *outfd,
- SHA1_CTX *ctx, struct got_repository *repo)
+ FILE *delta_cache, uint8_t *delta_cache_map, size_t delta_cache_size,
+ struct got_pack_meta *m, int *outfd, SHA1_CTX *ctx,
+ struct got_repository *repo)
{
const struct got_error *err = NULL;
struct got_deflate_checksum csum;
@@ -1740,6 +1760,16 @@ write_packed_object(off_t *packfile_size, FILE *packfile,
*packfile_size += m->delta_compressed_len;
free(m->delta_buf);
m->delta_buf = NULL;
+ } else if (delta_cache_map) {
+ err = deltahdr(packfile_size, ctx, packfile, m);
+ if (err)
+ goto done;
+ err = hcopy_mmap(delta_cache_map, m->delta_offset,
+ delta_cache_size, packfile, m->delta_compressed_len,
+ ctx);
+ if (err)
+ goto done;
+ *packfile_size += m->delta_compressed_len;
} else {
if (fseeko(delta_cache, m->delta_offset, SEEK_SET)
== -1) {
@@ -1779,12 +1809,37 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
size_t n;
off_t packfile_size = 0;
int outfd = -1;
+ int delta_cache_fd = -1;
+ uint8_t *delta_cache_map = NULL;
+ size_t delta_cache_size = 0;
SHA1Init(&ctx);
+#ifndef GOT_PACK_NO_MMAP
+ delta_cache_fd = dup(fileno(delta_cache));
+ if (delta_cache_fd != -1) {
+ struct stat sb;
+ if (fstat(delta_cache_fd, &sb) == -1) {
+ err = got_error_from_errno("fstat");
+ goto done;
+ }
+ if (sb.st_size > 0 && sb.st_size <= SIZE_MAX) {
+ delta_cache_map = mmap(NULL, sb.st_size,
+ PROT_READ, MAP_PRIVATE, delta_cache_fd, 0);
+ if (delta_cache_map == MAP_FAILED) {
+ if (errno != ENOMEM) {
+ err = got_error_from_errno("mmap");
+ goto done;
+ }
+ delta_cache_map = NULL; /* fallback on stdio */
+ } else
+ delta_cache_size = (size_t)sb.st_size;
+ }
+ }
+#endif
err = hwrite(packfile, "PACK", 4, &ctx);
if (err)
- return err;
+ goto done;
putbe32(buf, GOT_PACKFILE_VERSION);
err = hwrite(packfile, buf, 4, &ctx);
if (err)
@@ -1804,7 +1859,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
goto done;
m = deltify[i];
err = write_packed_object(&packfile_size, packfile,
- delta_cache, m, &outfd, &ctx, repo);
+ delta_cache, delta_cache_map, delta_cache_size,
+ m, &outfd, &ctx, repo);
if (err)
goto done;
}
@@ -1819,7 +1875,8 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
goto done;
m = reuse[i];
err = write_packed_object(&packfile_size, packfile,
- delta_cache, m, &outfd, &ctx, repo);
+ delta_cache, delta_cache_map, delta_cache_size,
+ m, &outfd, &ctx, repo);
if (err)
goto done;
}
@@ -1840,6 +1897,10 @@ genpack(uint8_t *pack_sha1, FILE *packfile, FILE *delta_cache,
done:
if (outfd != -1 && close(outfd) == -1 && err == NULL)
err = got_error_from_errno("close");
+ if (delta_cache_map && munmap(delta_cache_map, delta_cache_size) == -1)
+ err = got_error_from_errno("munmap");
+ if (delta_cache_fd != -1 && close(delta_cache_fd) == -1 && err == NULL)
+ err = got_error_from_errno("close");
return err;
}