Edit

kc3-lang/zlib/examples/gznorm.c

Branch :

  • Browse

    Commit

  • Author : Mark Adler
    Date : 2018-10-05 23:06:36
    Hash : 354fa43d
    Message : Add gznorm.c example, which normalizes gzip files.

  • examples/gznorm.c
  • /* gznorm.c -- normalize a gzip stream
     * Copyright (C) 2018 Mark Adler
     * For conditions of distribution and use, see copyright notice in zlib.h
     * Version 1.0  7 Oct 2018  Mark Adler */
    
    // gznorm takes a gzip stream, potentially containing multiple members, and
    // converts it to a gzip stream with a single member. In addition the gzip
    // header is normalized, removing the file name and time stamp, and setting the
    // other header contents (XFL, OS) to fixed values. gznorm does not recompress
    // the data, so it is fast, but no advantage is gained from the history that
    // could be available across member boundaries.
    
    #include <stdio.h>      // fread, fwrite, putc, fflush, ferror, fprintf,
                            // vsnprintf, stdout, stderr, NULL, FILE
    #include <stdlib.h>     // malloc, free
    #include <string.h>     // strerror
    #include <errno.h>      // errno
    #include <stdarg.h>     // va_list, va_start, va_end
    #include "zlib.h"       // inflateInit2, inflate, inflateReset, inflateEnd,
                            // z_stream, z_off_t, crc32_combine, Z_NULL, Z_BLOCK,
                            // Z_OK, Z_STREAM_END, Z_BUF_ERROR, Z_DATA_ERROR,
                            // Z_MEM_ERROR
    
    #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__)
    #  include <fcntl.h>
    #  include <io.h>
    #  define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
    #else
    #  define SET_BINARY_MODE(file)
    #endif
    
    #define local static
    
    // printf to an allocated string. Return the string, or NULL if the printf or
    // allocation fails.
    local char *aprintf(char *fmt, ...) {
        // Get the length of the result of the printf.
        va_list args;
        va_start(args, fmt);
        int len = vsnprintf(NULL, 0, fmt, args);
        va_end(args);
        if (len < 0)
            return NULL;
    
        // Allocate the required space and printf to it.
        char *str = malloc(len + 1);
        if (str == NULL)
            return NULL;
        va_start(args, fmt);
        vsnprintf(str, len + 1, fmt, args);
        va_end(args);
        return str;
    }
    
    // Return with an error, putting an allocated error message in *err. Doing an
    // inflateEnd() on an already ended state, or one with state set to Z_NULL, is
    // permitted.
    #define BYE(...) \
        do { \
            inflateEnd(&strm); \
            *err = aprintf(__VA_ARGS__); \
            return 1; \
        } while (0)
    
    // Chunk size for buffered reads and for decompression. Twice this many bytes
    // will be allocated on the stack by gzip_normalize(). Must fit in an unsigned.
    #define CHUNK 16384
    
    // Read a gzip stream from in and write an equivalent normalized gzip stream to
    // out. If given no input, an empty gzip stream will be written. If successful,
    // 0 is returned, and *err is set to NULL. On error, 1 is returned, where the
    // details of the error are returned in *err, a pointer to an allocated string.
    //
    // The input may be a stream with multiple gzip members, which is converted to
    // a single gzip member on the output. Each gzip member is decompressed at the
    // level of deflate blocks. This enables clearing the last-block bit, shifting
    // the compressed data to concatenate to the previous member's compressed data,
    // which can end at an arbitrary bit boundary, and identifying stored blocks in
    // order to resynchronize those to byte boundaries. The deflate compressed data
    // is terminated with a 10-bit empty fixed block. If any members on the input
    // end with a 10-bit empty fixed block, then that block is excised from the
    // stream. This avoids appending empty fixed blocks for every normalization,
    // and assures that gzip_normalize applied a second time will not change the
    // input. The pad bits after stored block headers and after the final deflate
    // block are all forced to zeros.
    local int gzip_normalize(FILE *in, FILE *out, char **err) {
        // initialize the inflate engine to process a gzip member
        z_stream strm;
        strm.zalloc = Z_NULL;
        strm.zfree = Z_NULL;
        strm.opaque = Z_NULL;
        strm.avail_in = 0;
        strm.next_in = Z_NULL;
        if (inflateInit2(&strm, 15 + 16) != Z_OK)
            BYE("out of memory");
    
        // State while processing the input gzip stream.
        enum {              // BETWEEN -> HEAD -> BLOCK -> TAIL -> BETWEEN -> ...
            BETWEEN,        // between gzip members (must end in this state)
            HEAD,           // reading a gzip header
            BLOCK,          // reading deflate blocks
            TAIL            // reading a gzip trailer
        } state = BETWEEN;              // current component being processed
        unsigned long crc = 0;          // accumulated CRC of uncompressed data
        unsigned long len = 0;          // accumulated length of uncompressed data
        unsigned long buf = 0;          // deflate stream bit buffer of num bits
        int num = 0;                    // number of bits in buf (at bottom)
    
        // Write a canonical gzip header (no mod time, file name, comment, extra
        // block, or extra flags, and OS is marked as unknown).
        fwrite("\x1f\x8b\x08\0\0\0\0\0\0\xff", 1, 10, out);
    
        // Process the gzip stream from in until reaching the end of the input,
        // encountering invalid input, or experiencing an i/o error.
        int more;                       // true if not at the end of the input
        do {
            // State inside this loop.
            unsigned char *put;         // next input buffer location to process
            int prev;                   // number of bits from previous block in
                                        // the bit buffer, or -1 if not at the
                                        // start of a block
            unsigned long long memb;    // uncompressed length of member
            size_t tail;                // number of trailer bytes read (0..8)
            unsigned long part;         // accumulated trailer component
    
            // Get the next chunk of input from in.
            unsigned char dat[CHUNK];
            strm.avail_in = fread(dat, 1, CHUNK, in);
            if (strm.avail_in == 0)
                break;
            more = strm.avail_in == CHUNK;
            strm.next_in = put = dat;
    
            // Run that chunk of input through the inflate engine to exhaustion.
            do {
                // At this point it is assured that strm.avail_in > 0.
    
                // Inflate until the end of a gzip component (header, deflate
                // block, trailer) is reached, or until all of the chunk is
                // consumed. The resulting decompressed data is discarded, though
                // the total size of the decompressed data in each member is
                // tracked, for the calculation of the total CRC.
                do {
                    // inflate and handle any errors
                    unsigned char scrap[CHUNK];
                    strm.avail_out = CHUNK;
                    strm.next_out = scrap;
                    int ret = inflate(&strm, Z_BLOCK);
                    if (ret == Z_MEM_ERROR)
                        BYE("out of memory");
                    if (ret == Z_DATA_ERROR)
                        BYE("input invalid: %s", strm.msg);
                    if (ret != Z_OK && ret != Z_BUF_ERROR && ret != Z_STREAM_END)
                        BYE("internal error");
    
                    // Update the number of uncompressed bytes generated in this
                    // member. The actual count (not modulo 2^32) is required to
                    // correctly compute the total CRC.
                    unsigned got = CHUNK - strm.avail_out;
                    memb += got;
                    if (memb < got)
                        BYE("overflow error");
    
                    // Continue to process this chunk until it is consumed, or
                    // until the end of a component (header, deflate block, or
                    // trailer) is reached.
                } while (strm.avail_out == 0 && (strm.data_type & 0x80) == 0);
    
                // Since strm.avail_in was > 0 for the inflate call, some input was
                // just consumed. It is therefore assured that put < strm.next_in.
    
                // Disposition the consumed component or part of a component.
                switch (state) {
                    case BETWEEN:
                        state = HEAD;
                        // Fall through to HEAD when some or all of the header is
                        // processed.
    
                    case HEAD:
                        // Discard the header.
                        if (strm.data_type & 0x80) {
                            // End of header reached -- deflate blocks follow.
                            put = strm.next_in;
                            prev = num;
                            memb = 0;
                            state = BLOCK;
                        }
                        break;
    
                    case BLOCK:
                        // Copy the deflate stream to the output, but with the
                        // last-block-bit cleared. Re-synchronize stored block
                        // headers to the output byte boundaries. The bytes at
                        // put..strm.next_in-1 is the compressed data that has been
                        // processed and is ready to be copied to the output.
    
                        // At this point, it is assured that new compressed data is
                        // available, i.e., put < strm.next_in. If prev is -1, then
                        // that compressed data starts in the middle of a deflate
                        // block. If prev is not -1, then the bits in the bit
                        // buffer, possibly combined with the bits in *put, contain
                        // the three-bit header of the new deflate block. In that
                        // case, prev is the number of bits from the previous block
                        // that remain in the bit buffer. Since num is the number
                        // of bits in the bit buffer, we have that num - prev is
                        // the number of bits from the new block currently in the
                        // bit buffer.
    
                        // If strm.data_type & 0xc0 is 0x80, then the last byte of
                        // the available compressed data includes the last bits of
                        // the end of a deflate block. In that case, that last byte
                        // also has strm.data_type & 0x1f bits of the next deflate
                        // block, in the range 0..7. If strm.data_type & 0xc0 is
                        // 0xc0, then the last byte of the compressed data is the
                        // end of the deflate stream, followed by strm.data_type &
                        // 0x1f pad bits, also in the range 0..7.
    
                        // Set bits to the number of bits not yet consumed from the
                        // last byte. If we are at the end of the block, bits is
                        // either the number of bits in the last byte belonging to
                        // the next block, or the number of pad bits after the
                        // final block. In either of those cases, bits is in the
                        // range 0..7.
                        ;                   // (required due to C syntax oddity)
                        int bits = strm.data_type & 0x1f;
    
                        if (prev != -1) {
                            // We are at the start of a new block. Clear the last
                            // block bit, and check for special cases. If it is a
                            // stored block, then emit the header and pad to the
                            // next byte boundary. If it is a final, empty fixed
                            // block, then excise it.
    
                            // Some or all of the three header bits for this block
                            // may already be in the bit buffer. Load any remaining
                            // header bits into the bit buffer.
                            if (num - prev < 3) {
                                buf += (unsigned long)*put++ << num;
                                num += 8;
                            }
    
                            // Set last to have a 1 in the position of the last
                            // block bit in the bit buffer.
                            unsigned long last = (unsigned long)1 << prev;
    
                            if (((buf >> prev) & 7) == 3) {
                                // This is a final fixed block. Load at least ten
                                // bits from this block, including the header, into
                                // the bit buffer. We already have at least three,
                                // so at most one more byte needs to be loaded.
                                if (num - prev < 10) {
                                    if (put == strm.next_in)
                                        // Need to go get and process more input.
                                        // We'll end up back here to finish this.
                                        break;
                                    buf += (unsigned long)*put++ << num;
                                    num += 8;
                                }
                                if (((buf >> prev) & 0x3ff) == 3) {
                                    // That final fixed block is empty. Delete it
                                    // to avoid adding an empty block every time a
                                    // gzip stream is normalized.
                                    num = prev;
                                    buf &= last - 1;    // zero the pad bits
                                }
                            }
                            else if (((buf >> prev) & 6) == 0) {
                                // This is a stored block. Flush to the next
                                // byte boundary after the three-bit header.
                                num = (prev + 10) & ~7;
                                buf &= last - 1;        // zero the pad bits
                            }
    
                            // Clear the last block bit.
                            buf &= ~last;
    
                            // Write out complete bytes in the bit buffer.
                            while (num >= 8) {
                                putc(buf, out);
                                buf >>= 8;
                                num -= 8;
                            }
    
                            // If no more bytes left to process, then we have
                            // consumed the byte that had bits from the next block.
                            if (put == strm.next_in)
                                bits = 0;
                        }
    
                        // We are done handling the deflate block header. Now copy
                        // all or almost all of the remaining compressed data that
                        // has been processed so far. Don't copy one byte at the
                        // end if it contains bits from the next deflate block or
                        // pad bits at the end of a deflate block.
    
                        // mix is 1 if we are at the end of a deflate block, and if
                        // some of the bits in the last byte follow this block. mix
                        // is 0 if we are in the middle of a deflate block, if the
                        // deflate block ended on a byte boundary, or if all of the
                        // compressed data processed so far has been consumed.
                        int mix = (strm.data_type & 0x80) && bits;
    
                        // Copy all of the processed compressed data to the output,
                        // except for the last byte if it contains bits from the
                        // next deflate block or pad bits at the end of the deflate
                        // stream. Copy the data after shifting in num bits from
                        // buf in front of it, leaving num bits from the end of the
                        // compressed data in buf when done.
                        unsigned char *end = strm.next_in - mix;
                        if (put < end) {
                            if (num)
                                // Insert num bits from buf before the data being
                                // copied.
                                do {
                                    buf += (unsigned)(*put++) << num;
                                    putc(buf, out);
                                    buf >>= 8;
                                } while (put < end);
                            else {
                                // No shifting needed -- write directly.
                                fwrite(put, 1, end - put, out);
                                put = end;
                            }
                        }
    
                        // Process the last processed byte if it wasn't written.
                        if (mix) {
                            // Load the last byte into the bit buffer.
                            buf += (unsigned)(*put++) << num;
                            num += 8;
    
                            if (strm.data_type & 0x40) {
                                // We are at the end of the deflate stream and
                                // there are bits pad bits. Discard the pad bits
                                // and write a byte to the output, if available.
                                // Leave the num bits left over in buf to prepend
                                // to the next deflate stream.
                                num -= bits;
                                if (num >= 8) {
                                    putc(buf, out);
                                    num -= 8;
                                    buf >>= 8;
                                }
    
                                // Force the pad bits in the bit buffer to zeros.
                                buf &= ((unsigned long)1 << num) - 1;
    
                                // Don't need to set prev here since going to TAIL.
                            }
                            else
                                // At the end of an internal deflate block. Leave
                                // the last byte in the bit buffer to examine on
                                // the next entry to BLOCK, when more bits from the
                                // next block will be available.
                                prev = num - bits;      // number of bits in buffer
                                                        // from current block
                        }
    
                        // Don't have a byte left over, so we are in the middle of
                        // a deflate block, or the deflate block ended on a byte
                        // boundary. Set prev appropriately for the next entry into
                        // BLOCK.
                        else if (strm.data_type & 0x80)
                            // The block ended on a byte boundary, so no header
                            // bits are in the bit buffer.
                            prev = num;
                        else
                            // In the middle of a deflate block, so no header here.
                            prev = -1;
    
                        // Check for the end of the deflate stream.
                        if ((strm.data_type & 0xc0) == 0xc0) {
                            // That ends the deflate stream on the input side, the
                            // pad bits were discarded, and any remaining bits from
                            // the last block in the stream are saved in the bit
                            // buffer to prepend to the next stream. Process the
                            // gzip trailer next.
                            tail = 0;
                            part = 0;
                            state = TAIL;
                        }
                        break;
    
                    case TAIL:
                        // Accumulate available trailer bytes to update the total
                        // CRC and the total uncompressed length.
                        do {
                            part = (part >> 8) + ((unsigned long)(*put++) << 24);
                            tail++;
                            if (tail == 4) {
                                // Update the total CRC.
                                z_off_t len2 = memb;
                                if (len2 < 0 || (unsigned long long)len2 != memb)
                                    BYE("overflow error");
                                crc = crc ? crc32_combine(crc, part, len2) : part;
                                part = 0;
                            }
                            else if (tail == 8) {
                                // Update the total uncompressed length. (It's ok
                                // if this sum is done modulo 2^32.)
                                len += part;
    
                                // At the end of a member. Set up to inflate an
                                // immediately following gzip member. (If we made
                                // it this far, then the trailer was valid.)
                                if (inflateReset(&strm) != Z_OK)
                                    BYE("internal error");
                                state = BETWEEN;
                                break;
                            }
                        } while (put < strm.next_in);
                        break;
                }
    
                // Process the input buffer until completely consumed.
            } while (strm.avail_in > 0);
    
            // Process input until end of file, invalid input, or i/o error.
        } while (more);
    
        // Done with the inflate engine.
        inflateEnd(&strm);
    
        // Verify the validity of the input.
        if (state != BETWEEN)
            BYE("input invalid: incomplete gzip stream");
    
        // Write the remaining deflate stream bits, followed by a terminating
        // deflate fixed block.
        buf += (unsigned long)3 << num;
        putc(buf, out);
        putc(buf >> 8, out);
        if (num > 6)
            putc(0, out);
    
        // Write the gzip trailer, which is the CRC and the uncompressed length
        // modulo 2^32, both in little-endian order.
        putc(crc, out);
        putc(crc >> 8, out);
        putc(crc >> 16, out);
        putc(crc >> 24, out);
        putc(len, out);
        putc(len >> 8, out);
        putc(len >> 16, out);
        putc(len >> 24, out);
        fflush(out);
    
        // Check for any i/o errors.
        if (ferror(in) || ferror(out))
            BYE("i/o error: %s", strerror(errno));
    
        // All good!
        *err = NULL;
        return 0;
    }
    
    // Normalize the gzip stream on stdin, writing the result to stdout.
    int main(void) {
        // Avoid end-of-line conversions on evil operating systems.
        SET_BINARY_MODE(stdin);
        SET_BINARY_MODE(stdout);
    
        // Normalize from stdin to stdout, returning 1 on error, 0 if ok.
        char *err;
        int ret = gzip_normalize(stdin, stdout, &err);
        if (ret)
            fprintf(stderr, "gznorm error: %s\n", err);
        free(err);
        return ret;
    }