Commit 55fdd2574a8da106021dd4857d3727185bda6c99
2020-03-18T16:13:43
do not buffer more data than necessary in got_inflate_to_mem_fd()
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122
diff --git a/lib/got_lib_inflate.h b/lib/got_lib_inflate.h
index e97951c..1d8212e 100644
--- a/lib/got_lib_inflate.h
+++ b/lib/got_lib_inflate.h
@@ -40,7 +40,7 @@ void got_inflate_end(struct got_inflate_buf *);
const struct got_error *got_inflate_to_mem(uint8_t **, size_t *, size_t *,
FILE *);
const struct got_error *got_inflate_to_mem_fd(uint8_t **, size_t *, size_t *,
- uint32_t *, int);
+ uint32_t *, size_t, int);
const struct got_error *got_inflate_to_mem_mmap(uint8_t **, size_t *, uint8_t *,
size_t, size_t);
const struct got_error *got_inflate_to_file(size_t *, FILE *, FILE *);
diff --git a/lib/inflate.c b/lib/inflate.c
index adb65f7..e1e7cd5 100644
--- a/lib/inflate.c
+++ b/lib/inflate.c
@@ -311,23 +311,27 @@ done:
const struct got_error *
got_inflate_to_mem_fd(uint8_t **outbuf, size_t *outlen,
- size_t *consumed_total, uint32_t *input_crc, int infd)
+ size_t *consumed_total, uint32_t *input_crc, size_t expected_size, int infd)
{
const struct got_error *err;
size_t avail, consumed;
struct got_inflate_buf zb;
void *newbuf;
int nbuf = 1;
+ size_t bufsize = GOT_INFLATE_BUFSIZE;
+ /* Optimize buffer size in case short reads should suffice. */
+ if (expected_size > 0 && expected_size < bufsize)
+ bufsize = expected_size;
+
if (outbuf) {
- *outbuf = malloc(GOT_INFLATE_BUFSIZE);
+ *outbuf = malloc(bufsize);
if (*outbuf == NULL)
return got_error_from_errno("malloc");
err = got_inflate_init(&zb, *outbuf, GOT_INFLATE_BUFSIZE,
input_crc);
} else
- err = got_inflate_init(&zb, NULL, GOT_INFLATE_BUFSIZE,
- input_crc);
+ err = got_inflate_init(&zb, NULL, bufsize, input_crc);
if (err)
goto done;
diff --git a/lib/pack.c b/lib/pack.c
index 28dd529..e5817bc 100644
--- a/lib/pack.c
+++ b/lib/pack.c
@@ -720,7 +720,7 @@ read_delta_data(uint8_t **delta_buf, size_t *delta_len,
if (lseek(pack->fd, delta_data_offset, SEEK_SET) == -1)
return got_error_from_errno("lseek");
err = got_inflate_to_mem_fd(delta_buf, delta_len, NULL,
- NULL, pack->fd);
+ NULL, 0, pack->fd);
}
return err;
}
@@ -1101,7 +1101,8 @@ dump_delta_chain_to_file(size_t *result_size, struct got_delta_chain *deltas,
pack->filesize - mapoff);
} else
err = got_inflate_to_mem_fd(&base_buf,
- &base_bufsz, NULL, NULL, pack->fd);
+ &base_bufsz, NULL, NULL, max_size,
+ pack->fd);
}
if (err)
goto done;
@@ -1248,7 +1249,8 @@ got_pack_dump_delta_chain_to_mem(uint8_t **outbuf, size_t *outlen,
goto done;
}
err = got_inflate_to_mem_fd(&base_buf,
- &base_bufsz, NULL, NULL, pack->fd);
+ &base_bufsz, NULL, NULL, max_size,
+ pack->fd);
}
if (err)
goto done;
@@ -1369,7 +1371,7 @@ got_packfile_extract_object_to_mem(uint8_t **buf, size_t *len,
if (lseek(pack->fd, obj->pack_offset, SEEK_SET) == -1)
return got_error_from_errno("lseek");
err = got_inflate_to_mem_fd(buf, len, NULL, NULL,
- pack->fd);
+ obj->size, pack->fd);
}
} else
err = got_pack_dump_delta_chain_to_mem(buf, len, &obj->deltas,
diff --git a/libexec/got-index-pack/got-index-pack.c b/libexec/got-index-pack/got-index-pack.c
index 4593055..fc35ff4 100644
--- a/libexec/got-index-pack/got-index-pack.c
+++ b/libexec/got-index-pack/got-index-pack.c
@@ -180,7 +180,7 @@ read_packed_object(struct got_pack *pack, struct got_indexed_object *obj)
case GOT_OBJ_TYPE_TAG:
/* XXX TODO reading large objects into memory is bad! */
err = got_inflate_to_mem_fd(&data, &datalen, &obj->len,
- &obj->crc, pack->fd);
+ &obj->crc, obj->size, pack->fd);
if (err)
break;
SHA1Init(&ctx);
@@ -213,7 +213,7 @@ read_packed_object(struct got_pack *pack, struct got_indexed_object *obj)
obj->crc = crc32(obj->crc, obj->ref_id.sha1,
SHA1_DIGEST_LENGTH);
err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
- &obj->crc, pack->fd);
+ &obj->crc, obj->size, pack->fd);
if (err)
break;
obj->len += SHA1_DIGEST_LENGTH;
@@ -235,7 +235,7 @@ read_packed_object(struct got_pack *pack, struct got_indexed_object *obj)
break;
err = got_inflate_to_mem_fd(NULL, &datalen, &obj->len,
- &obj->crc, pack->fd);
+ &obj->crc, obj->size, pack->fd);
if (err)
break;
obj->len += obj->base_offsetlen;