#include #include #include #include #include #include #include #include #include #include #include "espfsformat.h" #include "heatshrink_encoder.h" #include "espfs.h" #define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif" // static variables static int s_outFd = 1; struct InputFileLinkedListEntry; struct InputFileLinkedListEntry { char *name; struct InputFileLinkedListEntry *next; }; static struct InputFileLinkedListEntry *s_inputFiles = NULL; static struct InputFileLinkedListEntry *s_lastInputFile = NULL; static char **s_gzipExtensions = NULL; static bool s_gzipAll = false; // impls to satisfy defs in the config header void *httpdPlatMalloc(size_t len) { return malloc(len); } void httpdPlatFree(void *ptr) { free(ptr); } size_t compressHeatshrink(uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level) { uint8_t *inp = in; uint8_t *outp = out; size_t len; int ws[] = {5, 6, 8, 11, 13}; int ls[] = {3, 3, 4, 4, 4}; HSE_poll_res pres; HSE_sink_res sres; size_t r; if (level == -1) { level = 8; } level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4 heatshrink_encoder *enc = heatshrink_encoder_alloc(ws[level], ls[level]); if (enc == NULL) { perror("allocating mem for heatshrink"); exit(1); } //Save encoder parms as first byte *outp = (ws[level] << 4) | ls[level]; outp++; outcap--; r = 1; do { if (insize > 0) { sres = heatshrink_encoder_sink(enc, inp, insize, &len); if (sres != HSER_SINK_OK) { break; } inp += len; insize -= len; if (insize == 0) { heatshrink_encoder_finish(enc); } } do { pres = heatshrink_encoder_poll(enc, outp, outcap, &len); if (pres != HSER_POLL_MORE && pres != HSER_POLL_EMPTY) { break; } outp += len; outcap -= len; r += len; } while (pres == HSER_POLL_MORE); } while (insize != 0); if (insize != 0) { fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", (int)insize, sres, pres); exit(1); } heatshrink_encoder_free(enc); return r; } size_t compressGzip(uint8_t *in, size_t insize, uint8_t *out, size_t outsize, int level) { z_stream stream; int zresult; stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; stream.next_in = in; stream.avail_in = insize; stream.next_out = out; stream.avail_out = outsize; // 31 -> 15 window bits + 16 for gzip zresult = deflateInit2(&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); if (zresult != Z_OK) { fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult); exit(1); } zresult = deflate(&stream, Z_FINISH); if (zresult != Z_STREAM_END) { fprintf(stderr, "Deflate failed with code %d\n", zresult); exit(1); } zresult = deflateEnd(&stream); if (zresult != Z_OK) { fprintf(stderr, "DeflateEnd failed with code %d\n", zresult); exit(1); } return stream.total_out; } bool shouldCompressGzip(const char *name) { if (!s_gzipExtensions) { return false; } if (s_gzipAll) { return true; } const char *ext = name + strlen(name); while (*ext != '.') { ext--; if (ext < name) { // no dot in file name -> no extension -> nothing to match against return false; } } ext++; int i = 0; while (s_gzipExtensions[i] != NULL) { if (strcasecmp(ext, s_gzipExtensions[i]) == 0) { return true; } i++; } return false; } int parseGzipExtensions(char *input) { char *token; char *extList = input; int count = 2; // one for first element, second for terminator // count elements while (*extList != 0) { if (*extList == ',') { count++; } extList++; } // split string extList = input; s_gzipExtensions = malloc(count * sizeof(char *)); count = 0; token = strtok(extList, ","); while (token) { s_gzipExtensions[count++] = token; token = strtok(NULL, ","); } // terminate list s_gzipExtensions[count] = NULL; return 1; } /** * Process a file. * * @param fd - filedes * @param name - filename to embed in the archive * @param compression_mode - compression mode * @param level - compression level for heatshrink, 1-9 * @param[out] compName - the used compression is output here (for debug print) * @return */ int handleFile(int fd, const char *name, int compression_mode, int level, const char **compName) { uint8_t *fdat = NULL, *cdat = NULL, *cdatbuf = NULL; uint32_t size, csize; EspFsHeader h; uint16_t realNameLen; uint8_t flags = 0; size = lseek(fd, 0, SEEK_END); fdat = malloc(size); lseek(fd, 0, SEEK_SET); read(fd, fdat, size); if (shouldCompressGzip(name)) { csize = size * 3; if (csize < 100) { // gzip has some headers that do not fit when trying to compress small files csize = 100; } // enlarge buffer if this is the case cdat = cdatbuf = malloc(csize); csize = compressGzip(fdat, size, cdat, csize, level); compression_mode = COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger flags = FLAG_GZIP; } else if (compression_mode == COMPRESS_NONE) { csize = size; cdat = fdat; } else if (compression_mode == COMPRESS_HEATSHRINK) { cdat = cdatbuf = malloc(size * 2); csize = compressHeatshrink(fdat, size, cdat, size * 2, level); } else { fprintf(stderr, "Unknown compression - %d\n", compression_mode); exit(1); } if (csize > size) { fprintf(stderr, "! Compression enbiggened %s, embed as plain\n", name); //Compressing enbiggened this file. Revert to uncompressed store. compression_mode = COMPRESS_NONE; csize = size; cdat = fdat; flags = 0; } //Fill header data h.magic = htole32(ESPFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24); h.flags = flags; h.compression = (int8_t) compression_mode; h.nameLen = realNameLen = strlen(name) + 1; // zero terminator uint32_t padbytes = 0; if (h.nameLen & 3) { //Round to next 32bit boundary padbytes = 4 - (h.nameLen & 3); h.nameLen += padbytes; // include the bytes in "name" to make parsing easier - these will be zeroed out, so the c-string remains the same. } h.nameLen = htole16(h.nameLen); h.fileLenComp = htole32(csize); h.fileLenDecomp = htole32(size); write(s_outFd, &h, sizeof(EspFsHeader)); write(s_outFd, name, realNameLen); if (padbytes) { write(s_outFd, "\0\0\0", padbytes); // these zeros are included in h.nameLen } write(s_outFd, cdat, csize); //Pad out to 32bit boundary - the parser does this automatically when walking over the archive. if (csize & 3) { padbytes = 4 - (csize & 3); write(s_outFd, "\0\0\0", padbytes); csize += padbytes; } free(fdat); if (cdatbuf) { // free the buffer allocated for compression output free(cdatbuf); } // debug outputs ... if (compName != NULL) { if (h.compression == COMPRESS_HEATSHRINK) { *compName = "heatshrink"; } else if (h.compression == COMPRESS_NONE) { if (h.flags & FLAG_GZIP) { *compName = "gzip"; } else { *compName = "none"; } } else { *compName = "unknown"; } } // get compression % (lower is better) return size ? (int) ((csize * 100) / size) : 100; } //Write final dummy header with FLAG_LASTFILE set. void finishArchive() { EspFsHeader h; h.magic = htole32(ESPFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24); h.flags = FLAG_LASTFILE; h.compression = COMPRESS_NONE; h.nameLen = 0; h.fileLenComp = 0; h.fileLenDecomp = 0; write(s_outFd, &h, sizeof(EspFsHeader)); } static size_t espfs_parse_filesize = -1; static int espfs_parse_fd = -1; void parseEspfsFileAndShowItsContents(const char *filename) { int rv; fprintf(stderr, "Parsing: %s\n", filename); FILE *f = fopen(filename, "r"); if (!f) { perror(filename); exit(1); } int fd = fileno(f); espfs_parse_filesize = lseek(fd, 0, SEEK_END); lseek(fd, 0, SEEK_SET); espfs_parse_fd = fd; rv = espFsInit(); if (rv != 0) { fprintf(stderr, "Fail to init FS\n"); exit(1); } EspFsWalk walk; espFsWalkInit(&walk); EspFsHeader header; uint32_t offset; char namebuf[1024]; while (espFsWalkNext(&walk, &header, namebuf, 1024, &offset)) { fprintf(stderr, "at %04x: \"%s\", flags: %02x, comp: %s, compLen: %d, plainLen: %d\n", offset, namebuf, header.flags, header.compression == 1 ? "HS" : "None", header.fileLenComp, header.fileLenDecomp); } fclose(f); } int httpdPlatEspfsRead(void *dest, uint32_t offset, size_t len) { fprintf(stderr, "FS read @ %d, len %d\n", offset, len); if (offset + len > espfs_parse_filesize) { fprintf(stderr, "Read out fo range!\n"); return -1; } lseek(espfs_parse_fd, offset, SEEK_SET); read(espfs_parse_fd, dest, len); return 0; } void queueInputFile(char *name) { fprintf(stderr, "INFILE: %s\n", name); struct InputFileLinkedListEntry *tmp = malloc(sizeof(struct InputFileLinkedListEntry)); tmp->name = strdup(name); tmp->next = NULL; if (s_lastInputFile == NULL) { s_inputFiles = tmp; s_lastInputFile = tmp; } else { s_lastInputFile->next = tmp; s_lastInputFile = tmp; } } int main(int argc, char **argv) { int f; char inputFileName[1024]; char *realName; struct stat statBuf; int serr; int rate; int err = 0; int compType; //default compression type - heatshrink int compLvl = -1; bool use_gzip = false; compType = COMPRESS_HEATSHRINK; int c; char *outfile = NULL; char *parseFile = NULL; while (1) { int option_index = 0; static struct option long_options[] = { {"parse", required_argument, 0, 'p'}, {"compress", required_argument, 0, 'c'}, {"gzip", no_argument, 0, 'z'}, {"gzip-all", no_argument, 0, 'G'}, {"level", required_argument, 0, 'l'}, {"gzip-exts", required_argument, 0, 'g'}, {"input", required_argument, 0, 'i'}, {"output", required_argument, 0, 'o'}, {"help", no_argument, 0, 'h'}, {0, 0, 0, 0} }; c = getopt_long(argc, argv, "c:l:g:zGhp:i:o:0123456789", long_options, &option_index); if (c == -1) { break; } switch (c) { case 'h': goto show_help; case 'z': use_gzip = true; break; case '0' ... '9': compLvl = c - '0'; break; case 'p': parseFile = strdup(optarg); break; case 'c': compType = atoi(optarg); break; case 'G': use_gzip = true; s_gzipAll = true; break; case 'g': use_gzip = true; if (!parseGzipExtensions(optarg)) { fprintf(stderr, "Bad gzip extension list: %s\n", optarg); err = 1; goto show_help; } break; case 'l': compLvl = atoi(optarg); if (compLvl < 1 || compLvl > 9) { fprintf(stderr, "Bad compression level: %d\n", compLvl); err = 1; goto show_help; } break; case 'i': queueInputFile(optarg); break; case 'o': outfile = strdup(optarg); break; case '?': goto show_help; default: fprintf(stderr, "Unknown option: %c\n", c); err = 1; goto show_help; } } if (parseFile) { parseEspfsFileAndShowItsContents(parseFile); exit(0); } if (s_gzipExtensions == NULL && use_gzip) { parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS)); } if (optind < argc) { while (optind < argc) { queueInputFile(argv[optind++]); } } if (!s_inputFiles) { fprintf(stderr, "Reading input file names from stdin\n"); while (fgets(inputFileName, sizeof(inputFileName), stdin)) { //Kill off '\n' at the end inputFileName[strlen(inputFileName) - 1] = 0; queueInputFile(inputFileName); } } FILE *outfp = NULL; if (outfile) { fprintf(stderr, "Writing to %s\n", outfile); outfp = fopen(outfile, "w+"); if (!outfp) { perror(outfile); return 1; } s_outFd = fileno(outfp); ftruncate(s_outFd, 0); } else { fprintf(stderr, "Writing to stdout\n\n"); } struct InputFileLinkedListEntry *entry = s_inputFiles; while (entry) { char *name = entry->name; //Only include files serr = stat(name, &statBuf); if ((serr == 0) && S_ISREG(statBuf.st_mode)) { //Strip off './' or '/' madness. realName = name; if (name[0] == '.' && name[1] == '/') { realName += 2; } if (realName[0] == '/') { realName++; } f = open(name, O_RDONLY); if (f > 0) { const char *compName = "unknown"; rate = handleFile(f, realName, compType, compLvl, &compName); fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); close(f); } else { perror(name); } } else if (serr != 0) { perror(name); } entry = entry->next; } finishArchive(); fsync(s_outFd); if (outfp) { fclose(outfp); } return 0; show_help: fprintf(stderr, "%s - Program to create espfs images\n", argv[0]); fprintf(stderr, "Options:\n"); fprintf(stderr, "[-p|--parse FILE]\n Parse an espfs file and show a list of its contents. No other options apply in this mode.\n"); fprintf(stderr, "[-c|--compress COMPRESSOR]\n 0 - None, 1 - Heatshrink (default)\n"); fprintf(stderr, "[-l|--level LEVEL] or [-0 through -9]\n compression level 1-9, higher is better but uses more RAM\n"); fprintf(stderr, "[-z|--gzip]\n use gzip for files with extensions matching "DEFAULT_GZIP_EXTS"\n"); fprintf(stderr, "[-Z|--gzip-all]\n use gzip for all files\n"); fprintf(stderr, "[-g|--gzip-exts GZIPPED_EXTENSIONS]\n use gzip for files with custom extensions, comma-separated\n"); fprintf(stderr, "[-i|--input FILE]\n Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin if not specified by args\n"); fprintf(stderr, "[-o|--output FILE]\n Output file name; if not specified, outputs to stdout\n"); fprintf(stderr, "[-h|--help\n Show help.\n\n"); exit(err); }