spritehttpd/espfsbuilder/main.c

#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zlib.h>
#include <getopt.h>
#include <stdbool.h>

#include "espfsformat.h"
#include "heatshrink_encoder.h"
#include "parsing.h"

#define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif"

struct InputFileLinkedListEntry;

struct InputFileLinkedListEntry {
  char *name;
  struct InputFileLinkedListEntry *next;
};

/// two ends of a linked list with input files
static struct InputFileLinkedListEntry *s_inputFiles = NULL;
static struct InputFileLinkedListEntry *s_lastInputFile = NULL;

/// Output file FD
static int s_outFd = 1;

/// Array of gzipped extensions, ends with a NULL pointer
static char **s_gzipExtensions = NULL;
/// Gzip all files
static bool s_gzipAll = false;

// impls to satisfy defs in the config header
void *httpdPlatMalloc(size_t len)
{
    return malloc(len);
}

void httpdPlatFree(void *ptr)
{
    free(ptr);
}

/**
 * Compress a file using Heatshrink
 *
 * @param[in] in - pointer to the uncompressed input
 * @param insize - len of the uncompressed input
 * @param[out] out - destination buffer for the compressed data
 * @param outcap - capacity of the output buffer
 * @param level - compression level, 1-9; -1 for default.
 * @return actual length of the compressed data
 */
size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
{
    const uint8_t *inp = in;
    uint8_t *outp = out;
    size_t len;
    int ws[] = {5, 6, 8, 11, 13};
    int ls[] = {3, 3, 4, 4, 4};
    HSE_poll_res pres = 0;
    HSE_sink_res sres = 0;
    size_t r;
    if (level == -1) { level = 8; }
    level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4
    heatshrink_encoder *enc = heatshrink_encoder_alloc(ws[level], ls[level]);
    if (enc == NULL) {
        perror("allocating mem for heatshrink");
        exit(1);
    }
    //Save encoder parms as first byte
    *outp = (ws[level] << 4) | ls[level];
    outp++;
    outcap--;

    r = 1;
    do {
        if (insize > 0) {
            sres = heatshrink_encoder_sink(enc, inp, insize, &len);
            if (sres != HSER_SINK_OK) { break; }
            inp += len;
            insize -= len;
            if (insize == 0) { heatshrink_encoder_finish(enc); }
        }
        do {
            pres = heatshrink_encoder_poll(enc, outp, outcap, &len);
            if (pres != HSER_POLL_MORE && pres != HSER_POLL_EMPTY) { break; }
            outp += len;
            outcap -= len;
            r += len;
        } while (pres == HSER_POLL_MORE);
    } while (insize != 0);

    if (insize != 0) {
        fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", (int) insize, sres, pres);
        exit(1);
    }

    heatshrink_encoder_free(enc);
    return r;
}

/**
 * Compress a file using Gzip
 *
 * @param[in] in - pointer to the uncompressed input
 * @param insize - len of the uncompressed input
 * @param[out] out - destination buffer for the compressed data
 * @param outcap - capacity of the output buffer
 * @param level - compression level, 1-9; -1 for default.
 * @return actual length of the compressed data
 */
size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
{
    z_stream stream;
    int zresult;

    stream.zalloc = Z_NULL;
    stream.zfree = Z_NULL;
    stream.opaque = Z_NULL;
    stream.next_in = in;
    stream.avail_in = insize;
    stream.next_out = out;
    stream.avail_out = outcap;
    // 31 -> 15 window bits + 16 for gzip
    zresult = deflateInit2(&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
    if (zresult != Z_OK) {
        fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult);
        exit(1);
    }

    zresult = deflate(&stream, Z_FINISH);
    if (zresult != Z_STREAM_END) {
        fprintf(stderr, "Deflate failed with code %d\n", zresult);
        exit(1);
    }

    zresult = deflateEnd(&stream);
    if (zresult != Z_OK) {
        fprintf(stderr, "DeflateEnd failed with code %d\n", zresult);
        exit(1);
    }

    return stream.total_out;
}

/**
 * Check if a file name should be compressed by gzip
 *
 * @param name - file name
 * @return true if should compress
 */
bool shouldCompressGzip(const char *name)
{
    if (!s_gzipExtensions) { return false; }
    if (s_gzipAll) { return true; }

    const char *ext = name + strlen(name);
    while (*ext != '.') {
        ext--;
        if (ext < name) {
            // no dot in file name -> no extension -> nothing to match against
            return false;
        }
    }
    ext++;

    int i = 0;
    while (s_gzipExtensions[i] != NULL) {
        if (strcasecmp(ext, s_gzipExtensions[i]) == 0) {
            return true;
        }
        i++;
    }

    return false;
}

/**
 * Parse a list of gzipped extensions
 *
 * @param input - list of comma-separated extensions, e.g. "jpg,png"
 * @return
 */
void parseGzipExtensions(char *input)
{
    char *token;
    char *extList = input;
    int count = 2; // one for first element, second for terminator

    // count elements
    while (*extList != 0) {
        if (*extList == ',') { count++; }
        extList++;
    }

    // split string
    extList = input;
    s_gzipExtensions = malloc(count * sizeof(char *));
    count = 0;
    token = strtok(extList, ",");
    while (token) {
        s_gzipExtensions[count++] = token;
        token = strtok(NULL, ",");
    }
    // terminate list
    s_gzipExtensions[count] = NULL;
}

/**
 * Process a file.
 *
 * @param fd - filedes
 * @param[in] name - filename to embed in the archive
 * @param compression_mode - compression mode
 * @param level - compression level for heatshrink, 1-9
 * @param[out] compName - the used compression is output here (for debug print)
 * @return - size of the output, in percent (100% = no compression)
 */
int handleFile(int fd, const char *name, int compression_mode, int level, const char **compName)
{
    uint8_t *fdat = NULL, *cdat = NULL, *cdatbuf = NULL;
    uint32_t size, csize;
    EspFsHeader h;
    uint16_t realNameLen;
    uint8_t flags = 0;
    size = lseek(fd, 0, SEEK_END);
    fdat = malloc(size);
    lseek(fd, 0, SEEK_SET);
    read(fd, fdat, size);

    if (shouldCompressGzip(name)) {
        csize = size * 3;
        if (csize < 100) { // gzip has some headers that do not fit when trying to compress small files
            csize = 100;
        } // enlarge buffer if this is the case
        cdat = cdatbuf = malloc(csize);
        csize = compressGzip(fdat, size, cdat, csize, level);
        compression_mode = COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger
        flags = FLAG_GZIP;
    } else if (compression_mode == COMPRESS_NONE) {
        csize = size;
        cdat = fdat;
    } else if (compression_mode == COMPRESS_HEATSHRINK) {
        cdat = cdatbuf = malloc(size * 2);
        csize = compressHeatshrink(fdat, size, cdat, size * 2, level);
    } else {
        fprintf(stderr, "Unknown compression - %d\n", compression_mode);
        exit(1);
    }

    if (csize > size) {
        fprintf(stderr, "! Compression enbiggened %s, embed as plain\n", name);
        //Compressing enbiggened this file. Revert to uncompressed store.
        compression_mode = COMPRESS_NONE;
        csize = size;
        cdat = fdat;
        flags = 0;
    }

    //Fill header data
    h.magic = htole32(ESPFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
    h.flags = flags;
    h.compression = (int8_t) compression_mode;
    h.nameLen = realNameLen = strlen(name) + 1; // zero terminator
    uint32_t padbytes = 0;
    if (h.nameLen & 3) {
        //Round to next 32bit boundary
        padbytes = 4 - (h.nameLen & 3);
        h.nameLen += padbytes; // include the bytes in "name" to make parsing easier - these will be zeroed out, so the c-string remains the same.
    }
    h.nameLen = htole16(h.nameLen);
    h.fileLenComp = htole32(csize);
    h.fileLenDecomp = htole32(size);

    write(s_outFd, &h, sizeof(EspFsHeader));
    write(s_outFd, name, realNameLen);
    if (padbytes) {
        write(s_outFd, "\0\0\0", padbytes); // these zeros are included in h.nameLen
    }
    write(s_outFd, cdat, csize);
    //Pad out to 32bit boundary - the parser does this automatically when walking over the archive.
    if (csize & 3) {
        padbytes = 4 - (csize & 3);
        write(s_outFd, "\0\0\0", padbytes);
        csize += padbytes;
    }

    free(fdat);
    if (cdatbuf) {
        // free the buffer allocated for compression output
        free(cdatbuf);
    }

    // debug outputs ...

    if (compName != NULL) {
        if (h.compression == COMPRESS_HEATSHRINK) {
            *compName = "heatshrink";
        } else if (h.compression == COMPRESS_NONE) {
            if (h.flags & FLAG_GZIP) {
                *compName = "gzip";
            } else {
                *compName = "none";
            }
        } else {
            *compName = "unknown";
        }
    }

    // get compression % (lower is better)
    return size ? (int) ((csize * 100) / size) : 100;
}

/**
 * Write final dummy header with FLAG_LASTFILE set.
 */
void finishArchive()
{
    EspFsHeader h;
    h.magic = htole32(ESPFS_MAGIC);
    h.flags = FLAG_LASTFILE;
    h.compression = COMPRESS_NONE;
    h.nameLen = 0;
    h.fileLenComp = 0;
    h.fileLenDecomp = 0;
    write(s_outFd, &h, sizeof(EspFsHeader));
}

/**
 * Queue a file for adding to the archive.
 * Appends it to the `s_inputFiles` linked list.
 *
 * @param name - file name to add
 */
void queueInputFile(const char *name)
{
    fprintf(stderr, "INFILE: %s\n", name);

    struct InputFileLinkedListEntry *tmp = malloc(sizeof(struct InputFileLinkedListEntry));
    tmp->name = strdup(name);
    tmp->next = NULL;

    if (s_lastInputFile == NULL) {
        s_inputFiles = tmp;
        s_lastInputFile = tmp;
    } else {
        s_lastInputFile->next = tmp;
        s_lastInputFile = tmp;
    }
}

int main(int argc, char **argv)
{
    int f;
    char inputFileName[1024];
    struct stat statBuf;
    int serr;
    int rate;
    int err = 0;
    int compType;  //default compression type - heatshrink
    int compLvl = -1;
    bool use_gzip = false;

    compType = COMPRESS_HEATSHRINK;

    int c;
    char *outfile = NULL;
    char *parseFile = NULL;
    char *stripPath = NULL;
    char *extractFile = NULL;

    while (1) {
        int option_index = 0;
        static struct option long_options[] = {
                {"parse",      required_argument, 0, 'p'},
                {"extract",    required_argument, 0, 'e'},
                {"compress",   required_argument, 0, 'c'},
                {"gzip",       no_argument,       0, 'z'},
                {"gzip-all",   no_argument,       0, 'G'},
                {"level",      required_argument, 0, 'l'},
                {"gzip-exts",  required_argument, 0, 'g'},
                {"input",      required_argument, 0, 'i'},
                {"output",     required_argument, 0, 'o'},
                {"strip-path", required_argument, 0, 'S'},
                {"help",       no_argument,       0, 'h'},
                {0, 0,                            0, 0}
        };

        c = getopt_long(argc, argv, "c:l:g:zGS:e:hp:i:o:0123456789",
                        long_options, &option_index);
        if (c == -1) {
            break;
        }

        switch (c) {
            case 'h':
                goto show_help;

            case 'z':
                use_gzip = true;
                break;

            case '0' ... '9':
                compLvl = c - '0';
                break;

            case 'p':
                parseFile = strdup(optarg);
                break;

            case 'e':
                extractFile = strdup(optarg);
                break;

            case 'S':
                stripPath = strdup(optarg);
                break;

            case 'c':
                compType = atoi(optarg);
                break;

            case 'G':
                use_gzip = true;
                s_gzipAll = true;
                break;

            case 'g':
                use_gzip = true;
                parseGzipExtensions(optarg);
                break;

            case 'l':
                compLvl = atoi(optarg);
                if (compLvl < 1 || compLvl > 9) {
                    fprintf(stderr, "Bad compression level: %d\n", compLvl);
                    err = 1;
                    goto show_help;
                }
                break;

            case 'i':
                queueInputFile(optarg);
                break;

            case 'o':
                outfile = strdup(optarg);
                break;

            case '?':
                goto show_help;

            default:
                fprintf(stderr, "Unknown option: %c\n", c);
                err = 1;
                goto show_help;
        }
    }

    FILE *outfp = NULL;
    if (outfile) {
        fprintf(stderr, "Writing to %s\n", outfile);
        outfp = fopen(outfile, "w+");
        if (!outfp) {
            perror(outfile);
            return 1;
        }
        s_outFd = fileno(outfp);
        ftruncate(s_outFd, 0);
    } else {
        fprintf(stderr, "Writing to stdout\n\n");
    }

    if (parseFile) {
        parseEspfsImage(parseFile, extractFile, s_outFd);
        exit(0);
    }

    if (s_gzipExtensions == NULL && use_gzip) {
        parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS));
    }

    if (optind < argc) {
        while (optind < argc) {
            queueInputFile(argv[optind++]);
        }
    }

    if (!s_inputFiles) {
        fprintf(stderr, "Reading input file names from stdin\n");
        while (fgets(inputFileName, sizeof(inputFileName), stdin)) {
            //Kill off '\n' at the end
            inputFileName[strlen(inputFileName) - 1] = 0;
            queueInputFile(inputFileName);
        }
    }

    struct InputFileLinkedListEntry *entry = s_inputFiles;
    while (entry) {
        char *name = entry->name;
        //Only include files
        serr = stat(name, &statBuf);
        if ((serr == 0) && S_ISREG(statBuf.st_mode)) {
            //Strip off './' or '/' madness.
            char *embeddedName = name;
            f = open(name, O_RDONLY);

            // relative path starting with ./, remove that
            if (embeddedName[0] == '.' && embeddedName[1] == '/') {
                embeddedName += 2;
            }
            // remove prefix
            if (stripPath && 0 == strncmp(embeddedName, stripPath, strlen(stripPath))) {
                embeddedName += strlen(stripPath);
            }
            // remove leading slash, if any
            if (embeddedName[0] == '/') { embeddedName++; }

            if (f > 0) {
                const char *compName = "unknown";
                rate = handleFile(f, embeddedName, compType, compLvl, &compName);
                fprintf(stderr, "%s (%d%%, %s)\n", embeddedName, rate, compName);
                close(f);
            } else {
                perror(name);
            }
        } else if (serr != 0) {
            perror(name);
        }

        entry = entry->next;
    }

    finishArchive();
    fsync(s_outFd);

    if (outfp) {
        fclose(outfp);
    }

    return 0;


    show_help:
    fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
    fprintf(stderr, "Options:\n");
    fprintf(stderr, "[-p|--parse FILE]\n    Parse an espfs file and show a list of its contents. No other options apply in this mode.\n");
    fprintf(stderr, "[-e|--extract FILE]\n    Extract a file with the given name from the parsed file (-p)\n");
    fprintf(stderr, "[-S|--strip-path PATH]\n    Remove the given path from input file names before packing\n");
    fprintf(stderr, "[-c|--compress COMPRESSOR]\n    0 - None, 1 - Heatshrink (default)\n");
    fprintf(stderr, "[-l|--level LEVEL] or [-0 through -9]\n    compression level 1-9, higher is better but uses more RAM\n");
    fprintf(stderr, "[-z|--gzip]\n    use gzip for files with extensions matching "DEFAULT_GZIP_EXTS"\n");
    fprintf(stderr, "[-G|--gzip-all]\n    use gzip for all files\n");
    fprintf(stderr, "[-g|--gzip-exts GZIPPED_EXTENSIONS]\n    use gzip for files with custom extensions, comma-separated\n");
    fprintf(stderr, "[-i|--input FILE]\n    Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin if not specified by args\n");
    fprintf(stderr, "[-o|--output FILE]\n    Output file name; if not specified, outputs to stdout\n");
    fprintf(stderr, "[-h|--help\n    Show help.\n\n");
    exit(err);
}