Commit 91b40e30e0dbff0c8a1546a02fb784fa8007a91b
2021-05-21T20:20:28
add checksum support to got_deflate_to_file() This will eventually be used by 'gotadmin pack'. Checksum init and finalization will need to be done by the caller since many objects will be written out in compressed form while we are computing checksums across the entire pack file. ok millert, naddy
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
diff --git a/lib/deflate.c b/lib/deflate.c
index c7771b4..a9a36a6 100644
--- a/lib/deflate.c
+++ b/lib/deflate.c
@@ -35,7 +35,8 @@
#endif
const struct got_error *
-got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
+got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize,
+ struct got_deflate_checksum *csum)
{
const struct got_error *err = NULL;
int zerr;
@@ -74,12 +75,23 @@ got_deflate_init(struct got_deflate_buf *zb, uint8_t *outbuf, size_t bufsize)
} else
zb->outbuf = outbuf;
+ zb->csum = csum;
done:
if (err)
got_deflate_end(zb);
return err;
}
+static void
+csum_output(struct got_deflate_checksum *csum, const char *buf, size_t len)
+{
+ if (csum->output_crc)
+ *csum->output_crc = crc32(*csum->output_crc, buf, len);
+
+ if (csum->output_sha1)
+ SHA1Update(csum->output_sha1, buf, len);
+}
+
const struct got_error *
got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)
{
@@ -92,6 +104,9 @@ got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)
*outlenp = 0;
do {
+ char *csum_out = NULL;
+ size_t csum_avail = 0;
+
if (z->avail_in == 0) {
size_t n = fread(zb->inbuf, 1, zb->inlen, f);
if (n == 0) {
@@ -104,7 +119,15 @@ got_deflate_read(struct got_deflate_buf *zb, FILE *f, size_t *outlenp)
z->next_in = zb->inbuf;
z->avail_in = n;
}
+ if (zb->csum) {
+ csum_out = z->next_out;
+ csum_avail = z->avail_out;
+ }
ret = deflate(z, Z_NO_FLUSH);
+ if (zb->csum) {
+ csum_output(zb->csum, csum_out,
+ csum_avail - z->avail_out);
+ }
} while (ret == Z_OK && z->avail_out > 0);
if (ret == Z_OK) {
@@ -129,13 +152,14 @@ got_deflate_end(struct got_deflate_buf *zb)
}
const struct got_error *
-got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
+got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile,
+ struct got_deflate_checksum *csum)
{
const struct got_error *err;
size_t avail;
struct got_deflate_buf zb;
- err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE);
+ err = got_deflate_init(&zb, NULL, GOT_DEFLATE_BUFSIZE, csum);
if (err)
goto done;
@@ -157,8 +181,6 @@ got_deflate_to_file(size_t *outlen, FILE *infile, FILE *outfile)
} while (zb.flags & GOT_DEFLATE_F_HAVE_MORE);
done:
- if (err == NULL)
- rewind(outfile);
got_deflate_end(&zb);
return err;
}
diff --git a/lib/got_lib_deflate.h b/lib/got_lib_deflate.h
index 1777eab..c359d7c 100644
--- a/lib/got_lib_deflate.h
+++ b/lib/got_lib_deflate.h
@@ -14,6 +14,14 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
+struct got_deflate_checksum {
+ /* If not NULL, mix output bytes into this CRC checksum. */
+ uint32_t *output_crc;
+
+ /* If not NULL, mix output bytes into this SHA1 context. */
+ SHA1_CTX *output_sha1;
+};
+
struct got_deflate_buf {
z_stream z;
char *inbuf;
@@ -23,13 +31,15 @@ struct got_deflate_buf {
int flags;
#define GOT_DEFLATE_F_HAVE_MORE 0x01
#define GOT_DEFLATE_F_OWN_OUTBUF 0x02
+ struct got_deflate_checksum *csum;
};
#define GOT_DEFLATE_BUFSIZE 8192
const struct got_error *got_deflate_init(struct got_deflate_buf *, uint8_t *,
- size_t);
+ size_t, struct got_deflate_checksum *);
const struct got_error *got_deflate_read(struct got_deflate_buf *, FILE *,
size_t *);
void got_deflate_end(struct got_deflate_buf *);
-const struct got_error *got_deflate_to_file(size_t *, FILE *, FILE *);
+const struct got_error *got_deflate_to_file(size_t *, FILE *, FILE *,
+ struct got_deflate_checksum *);
diff --git a/lib/object_create.c b/lib/object_create.c
index 5b84439..1598218 100644
--- a/lib/object_create.c
+++ b/lib/object_create.c
@@ -83,7 +83,7 @@ create_object_file(struct got_object_id *id, FILE *content,
goto done;
}
- err = got_deflate_to_file(&tmplen, content, tmpfile);
+ err = got_deflate_to_file(&tmplen, content, tmpfile, NULL);
if (err)
goto done;