Commit 39ff877fbb5335dfcf380e186a76b54074407112
2018-03-13T16:31:33
expand deltas in memory if result size is < 32MB
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
diff --git a/lib/delta.c b/lib/delta.c
index 537dfbc..f8ab225 100644
--- a/lib/delta.c
+++ b/lib/delta.c
@@ -233,6 +233,9 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
uint64_t base_size, result_size;
size_t remain, outsize = 0;
const uint8_t *p;
+ FILE *memstream = NULL;
+ char *memstream_buf = NULL;
+ size_t memstream_size = 0;
if (delta_len < GOT_DELTA_STREAM_LENGTH_MIN)
return got_error(GOT_ERR_BAD_DELTA);
@@ -251,6 +254,9 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
if (err)
return err;
+ if (result_size < GOT_DELTA_RESULT_SIZE_CACHED_MAX)
+ memstream = open_memstream(&memstream_buf, &memstream_size);
+
/* Decode and execute copy instructions from the delta stream. */
err = next_delta_byte(&p, &remain);
while (err == NULL && remain > 0) {
@@ -260,7 +266,8 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
err = parse_opcode(&offset, &len, &p, &remain);
if (err)
break;
- err = copy_from_base(base_file, offset, len, outfile);
+ err = copy_from_base(base_file, offset, len,
+ memstream ? memstream : outfile);
if (err == NULL) {
outsize += len;
if (remain > 0) {
@@ -277,7 +284,8 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
err = next_delta_byte(&p, &remain);
if (err)
break;
- err = copy_from_delta(&p, &remain, len, outfile);
+ err = copy_from_delta(&p, &remain, len,
+ memstream ? memstream : outfile);
if (err == NULL)
outsize += len;
}
@@ -286,6 +294,16 @@ got_delta_apply(FILE *base_file, const uint8_t *delta_buf,
if (outsize != result_size)
err = got_error(GOT_ERR_BAD_DELTA);
+ if (memstream != NULL) {
+ fclose(memstream);
+ if (err == NULL) {
+ size_t n;
+ n = fwrite(memstream_buf, 1, memstream_size, outfile);
+ if (n != memstream_size)
+ err = got_ferror(outfile, GOT_ERR_IO);
+ }
+ free(memstream_buf);
+ }
if (err == NULL)
rewind(outfile);
return err;
diff --git a/lib/got_delta_lib.h b/lib/got_delta_lib.h
index e34bc99..c0738c0 100644
--- a/lib/got_delta_lib.h
+++ b/lib/got_delta_lib.h
@@ -38,6 +38,12 @@ const struct got_error *got_delta_apply(FILE *, const uint8_t *, size_t,
FILE *);
/*
+ * The amount of result data we may keep in RAM while applying deltas.
+ * Data larger than this is written to disk during delta application (slow).
+ */
+#define GOT_DELTA_RESULT_SIZE_CACHED_MAX (32 * 1024 * 1024) /* bytes */
+
+/*
* Definitions for delta data streams.
*/