Commit 6395114c60289f3ac0f28c0fc4b5dfcef8690cdf
2022-05-31T08:44:55
avoid get_delta_chain_max_size() in dump_delta_chain_to_file()
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
diff --git a/lib/pack.c b/lib/pack.c
index 4bac59b..280287e 100644
--- a/lib/pack.c
+++ b/lib/pack.c
@@ -1257,8 +1257,10 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
const struct got_error *err = NULL;
struct got_delta *delta;
uint8_t *base_buf = NULL, *accum_buf = NULL, *delta_buf;
- size_t base_bufsz = 0, accum_size = 0, delta_len;
- uint64_t max_size;
+ size_t base_bufsz = 0, accum_bufsz = 0, accum_size = 0, delta_len;
+ /* We process small enough files entirely in memory for speed. */
+ const size_t max_bufsize = GOT_DELTA_RESULT_SIZE_CACHED_MAX;
+ uint64_t max_size = 0;
int n = 0;
*result_size = 0;
@@ -1266,25 +1268,14 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
if (STAILQ_EMPTY(&deltas->entries))
return got_error(GOT_ERR_BAD_DELTA_CHAIN);
- /* We process small enough files entirely in memory for speed. */
- err = got_pack_get_delta_chain_max_size(&max_size, deltas, pack);
- if (err)
- return err;
- if (max_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX) {
- accum_buf = malloc(max_size);
- if (accum_buf == NULL)
- return got_error_from_errno("malloc");
- base_file = NULL;
- accum_file = NULL;
- } else {
- if (fseeko(base_file, 0L, SEEK_SET) == -1)
- return got_error_from_errno("fseeko");
- if (fseeko(accum_file, 0L, SEEK_SET) == -1)
- return got_error_from_errno("fseeko");
- }
+ if (fseeko(base_file, 0L, SEEK_SET) == -1)
+ return got_error_from_errno("fseeko");
+ if (fseeko(accum_file, 0L, SEEK_SET) == -1)
+ return got_error_from_errno("fseeko");
/* Deltas are ordered in ascending order. */
STAILQ_FOREACH(delta, &deltas->entries, entry) {
+ uint64_t base_size, result_size = 0;
int cached = 1;
if (n == 0) {
size_t mapoff;
@@ -1311,7 +1302,9 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
goto done;
}
}
- if (base_file) {
+ if (delta->size > max_size)
+ max_size = delta->size;
+ if (max_size > max_bufsize) {
if (pack->map) {
mapoff = (size_t)delta_data_offset;
err = got_inflate_to_file_mmap(
@@ -1323,6 +1316,12 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
&base_bufsz, NULL, NULL, pack->fd,
base_file);
} else {
+ accum_buf = malloc(max_size);
+ if (accum_buf == NULL) {
+ err = got_error_from_errno("malloc");
+ goto done;
+ }
+ accum_bufsz = max_size;
if (pack->map) {
mapoff = (size_t)delta_data_offset;
err = got_inflate_to_mem_mmap(&base_buf,
@@ -1337,7 +1336,7 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
if (err)
goto done;
n++;
- if (base_file)
+ if (base_buf == NULL)
rewind(base_file);
continue;
}
@@ -1359,6 +1358,50 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
goto done;
}
}
+
+ err = got_delta_get_sizes(&base_size, &result_size,
+ delta_buf, delta_len);
+ if (err)
+ goto done;
+ if (base_size > max_size)
+ max_size = base_size;
+ if (result_size > max_size)
+ max_size = result_size;
+
+ if (base_buf && max_size > max_bufsize) {
+ /* Switch from buffers to temporary files. */
+ size_t w = fwrite(base_buf, 1, base_bufsz,
+ base_file);
+ if (w != base_bufsz) {
+ err = got_ferror(outfile, GOT_ERR_IO);
+ goto done;
+ }
+ free(base_buf);
+ base_buf = NULL;
+ free(accum_buf);
+ accum_buf = NULL;
+ }
+
+ if (base_buf && max_size > base_bufsz) {
+ uint8_t *p = realloc(base_buf, max_size);
+ if (p == NULL) {
+ err = got_error_from_errno("realloc");
+ goto done;
+ }
+ base_buf = p;
+ base_bufsz = max_size;
+ }
+
+ if (accum_buf && max_size > accum_bufsz) {
+ uint8_t *p = realloc(accum_buf, max_size);
+ if (p == NULL) {
+ err = got_error_from_errno("realloc");
+ goto done;
+ }
+ accum_buf = p;
+ accum_bufsz = max_size;
+ }
+
if (base_buf) {
err = got_delta_apply_in_mem(base_buf, base_bufsz,
delta_buf, delta_len, accum_buf,
@@ -1380,25 +1423,11 @@ got_pack_dump_delta_chain_to_file(size_t *result_size,
/* Accumulated delta becomes the new base. */
if (base_buf) {
uint8_t *tmp = accum_buf;
- /*
- * Base buffer switches roles with accumulation
- * buffer. Ensure it can hold the largest
- * result in the delta chain. The initial
- * allocation might have been smaller.
- */
- if (base_bufsz < max_size) {
- uint8_t *p;
- p = reallocarray(base_buf, 1, max_size);
- if (p == NULL) {
- err = got_error_from_errno(
- "reallocarray");
- goto done;
- }
- base_buf = p;
- base_bufsz = max_size;
- }
+ size_t tmp_size = accum_bufsz;
accum_buf = base_buf;
+ accum_bufsz = base_bufsz;
base_buf = tmp;
+ base_bufsz = tmp_size;
} else {
FILE *tmp = accum_file;
accum_file = base_file;