From 7fabf1110735eb78cc4a31c21bcc17a0f46e1d97 Mon Sep 17 00:00:00 2001 From: Jindra Dolezy Date: Sat, 11 Apr 2015 18:01:17 +0200 Subject: [PATCH] mkespfsimage is now able to gzip files with selected extensions --- espfs/espfsformat.h | 1 + espfs/mkespfsimage/main.c | 163 +++++++++++++++++++++++++++++++++++--- 2 files changed, 153 insertions(+), 11 deletions(-) diff --git a/espfs/espfsformat.h b/espfs/espfsformat.h index 325cad5..8ce5549 100644 --- a/espfs/espfsformat.h +++ b/espfs/espfsformat.h @@ -16,6 +16,7 @@ with the FLAG_LASTFILE flag set. #define FLAG_LASTFILE (1<<0) +#define FLAG_GZIP (1<<1) #define COMPRESS_NONE 0 #define COMPRESS_HEATSHRINK 1 #define ESPFS_MAGIC 0x73665345 diff --git a/espfs/mkespfsimage/main.c b/espfs/mkespfsimage/main.c index 921823a..64187ff 100644 --- a/espfs/mkespfsimage/main.c +++ b/espfs/mkespfsimage/main.c @@ -18,6 +18,13 @@ #include "heatshrink_encoder.h" #endif +//Gzip +#ifdef ESPFS_GZIP +// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" +// to install missing package. +#include +#endif + //Routines to convert host format to the endianness used in the xtensa short htoxs(short in) { @@ -83,18 +90,115 @@ size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int leve } #endif -int handleFile(int f, char *name, int compression, int level) { +#ifdef ESPFS_GZIP +size_t compressGzip(char *in, int insize, char *out, int outsize, int level) { + z_stream stream; + int zresult; + + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + stream.next_in = in; + stream.avail_in = insize; + stream.next_out = out; + stream.avail_out = outsize; + // 31 -> 15 window bits + 16 for gzip + zresult = deflateInit2 (&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY); + if (zresult != Z_OK) { + fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult); + exit(1); + } + + zresult = deflate(&stream, Z_FINISH); + if (zresult != Z_STREAM_END) { + fprintf(stderr, "Deflate failed with code %d\n", zresult); + exit(1); + } + + zresult = deflateEnd(&stream); + if (zresult != Z_OK) { + fprintf(stderr, "DeflateEnd failed with code %d\n", zresult); + exit(1); + } + + return stream.total_out; +} + +char **gzipExtensions = NULL; + +int shouldCompressGzip(char *name) { + char *ext = name + strlen(name); + while (*ext != '.') { + ext--; + if (ext < name) { + // no dot in file name -> no extension -> nothing to match against + return 0; + } + } + ext++; + + int i = 0; + while (gzipExtensions[i] != NULL) { + if (strcmp(ext,gzipExtensions[i]) == 0) { + return 1; + } + i++; + } + + return 0; +} + +int parseGzipExtensions(char *input) { + char *token; + char *extList = input; + int count = 2; // one for first element, second for terminator + + // count elements + while (*extList != 0) { + if (*extList == ',') count++; + extList++; + } + + // split string + extList = input; + gzipExtensions = malloc(count * sizeof(char*)); + count = 0; + token = strtok(extList, ","); + while (token) { + gzipExtensions[count++] = token; + token = strtok(NULL, ","); + } + // terminate list + gzipExtensions[count] = NULL; + + return 1; +} +#endif + +int handleFile(int f, char *name, int compression, int level, char **compName) { char *fdat, *cdat; off_t size, csize; EspFsHeader h; int nameLen; + int8_t flags = 0; size=lseek(f, 0, SEEK_END); fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0); if (fdat==MAP_FAILED) { perror("mmap"); return 0; } - + +#ifdef ESPFS_GZIP + if (shouldCompressGzip(name)) { + csize = size*3; + if (csize<100) // gzip has some headers that do not fit when trying to compress small files + csize = 100; // enlarge buffer if this is the case + cdat=malloc(csize); + csize=compressGzip(fdat, size, cdat, csize, level); + compression = COMPRESS_NONE; + flags = FLAG_GZIP; + } else +#endif if (compression==COMPRESS_NONE) { csize=size; cdat=fdat; @@ -113,20 +217,25 @@ int handleFile(int f, char *name, int compression, int level) { compression=COMPRESS_NONE; csize=size; cdat=fdat; + flags=0; } //Fill header data h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24); - h.flags=0; + h.flags=flags; h.compression=compression; - nameLen=strlen(name)+1; - if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary - h.nameLen=htoxs(nameLen); + h.nameLen=nameLen=strlen(name)+1; + if (h.nameLen&3) h.nameLen+=4-(h.nameLen&3); //Round to next 32bit boundary + h.nameLen=htoxs(h.nameLen); h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); write(1, &h, sizeof(EspFsHeader)); - write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer. + write(1, name, nameLen); + while (nameLen&3) { + write(1, "\000", 1); + nameLen++; + } write(1, cdat, csize); //Pad out to 32bit boundary while (csize&3) { @@ -134,6 +243,20 @@ int handleFile(int f, char *name, int compression, int level) { csize++; } munmap(fdat, size); + + if (compName != NULL) { + if (h.compression==COMPRESS_HEATSHRINK) { + *compName = "heatshrink"; + } else if (h.compression==COMPRESS_NONE) { + if (h.flags & FLAG_GZIP) { + *compName = "gzip"; + } else { + *compName = "none"; + } + } else { + *compName = "unknown"; + } + } return (csize*100)/size; } @@ -149,7 +272,6 @@ void finishArchive() { write(1, &h, sizeof(EspFsHeader)); } - int main(int argc, char **argv) { int f, x; char fileName[1024]; @@ -175,14 +297,29 @@ int main(int argc, char **argv) { compLvl=atoi(argv[x+1]); if (compLvl<1 || compLvl>9) err=1; x++; +#ifdef ESPFS_GZIP + } else if (strcmp(argv[x], "-g")==0 && argc>=x-2) { + if (!parseGzipExtensions(argv[x+1])) err=1; + x++; +#endif } else { err=1; } } +#ifdef ESPFS_GZIP + if (gzipExtensions == NULL) { + parseGzipExtensions(strdup("html,css,js")); + } +#endif + if (err) { fprintf(stderr, "%s - Program to create espfs images\n", argv[0]); - fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] > out.espfs\n", argv[0]); + fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] ", argv[0]); +#ifdef ESPFS_GZIP + fprintf(stderr, "[-g gzipped_extensions] "); +#endif + fprintf(stderr, "> out.espfs\n"); fprintf(stderr, "Compressors:\n"); #ifdef ESPFS_HEATSHRINK fprintf(stderr, "0 - None\n1 - Heatshrink(default)\n"); @@ -190,6 +327,9 @@ int main(int argc, char **argv) { fprintf(stderr, "0 - None(default)\n"); #endif fprintf(stderr, "\nCompression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n"); +#ifdef ESPFS_GZIP + fprintf(stderr, "\nGzipped extensions: list of comma separated, case sensitive file extensions \nthat will be gzipped. Defaults to 'html,css,js'\n"); +#endif exit(0); } @@ -205,8 +345,9 @@ int main(int argc, char **argv) { if (realName[0]=='/') realName++; f=open(fileName, O_RDONLY); if (f>0) { - rate=handleFile(f, realName, compType, compLvl); - fprintf(stderr, "%s (%d%%)\n", realName, rate); + char *compName = "unknown"; + rate=handleFile(f, realName, compType, compLvl, &compName); + fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); close(f); } else { perror(fileName);