mkespfsimage is now able to gzip files with selected extensions

espfs/espfsformat.h

@@ -16,6 +16,7 @@ with the FLAG_LASTFILE flag set.
 
 
 #define FLAG_LASTFILE (1<<0)
+#define FLAG_GZIP (1<<1)
 #define COMPRESS_NONE 0
 #define COMPRESS_HEATSHRINK 1
 #define ESPFS_MAGIC 0x73665345

espfs/mkespfsimage/main.c

@@ -18,6 +18,13 @@
 #include "heatshrink_encoder.h"
 #endif
 
+//Gzip
+#ifdef ESPFS_GZIP
+// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" 
+// to install missing package.
+#include <zlib.h>
+#endif
+
 
 //Routines to convert host format to the endianness used in the xtensa
 short htoxs(short in) {
@@ -83,11 +90,97 @@ size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int leve
 }
 #endif
 
-int handleFile(int f, char *name, int compression, int level) {
+#ifdef ESPFS_GZIP
+size_t compressGzip(char *in, int insize, char *out, int outsize, int level) {
+	z_stream stream;
+	int zresult;
+
+	stream.zalloc = Z_NULL;
+	stream.zfree  = Z_NULL;
+	stream.opaque = Z_NULL;
+	stream.next_in = in;
+	stream.avail_in = insize;
+	stream.next_out = out;
+	stream.avail_out = outsize;
+	// 31 -> 15 window bits + 16 for gzip
+	zresult = deflateInit2 (&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
+	if (zresult != Z_OK) {
+		fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult);
+		exit(1);
+	}
+
+	zresult = deflate(&stream, Z_FINISH);
+	if (zresult != Z_STREAM_END) {
+		fprintf(stderr, "Deflate failed with code %d\n", zresult);
+		exit(1);
+	}
+
+	zresult = deflateEnd(&stream);
+	if (zresult != Z_OK) {
+		fprintf(stderr, "DeflateEnd failed with code %d\n", zresult);
+		exit(1);
+	}
+
+	return stream.total_out;
+}
+
+char **gzipExtensions = NULL;
+
+int shouldCompressGzip(char *name) {
+	char *ext = name + strlen(name);
+	while (*ext != '.') {
+		ext--;
+		if (ext < name) {
+			// no dot in file name -> no extension -> nothing to match against
+			return 0;
+		}
+	}
+	ext++;
+
+	int i = 0;
+	while (gzipExtensions[i] != NULL) {
+		if (strcmp(ext,gzipExtensions[i]) == 0) {
+			return 1;
+		}
+		i++;
+	}
+
+	return 0;
+}
+
+int parseGzipExtensions(char *input) {
+	char *token;
+	char *extList = input;
+	int count = 2; // one for first element, second for terminator
+
+	// count elements
+	while (*extList != 0) {
+		if (*extList == ',') count++;
+		extList++;
+	}
+
+	// split string
+	extList = input;
+	gzipExtensions = malloc(count * sizeof(char*));
+	count = 0;
+	token = strtok(extList, ",");
+	while (token) {
+		gzipExtensions[count++] = token;
+		token = strtok(NULL, ",");
+	}
+	// terminate list
+	gzipExtensions[count] = NULL;
+
+	return 1;
+}
+#endif
+
+int handleFile(int f, char *name, int compression, int level, char **compName) {
 	char *fdat, *cdat;
 	off_t size, csize;
 	EspFsHeader h;
 	int nameLen;
+	int8_t flags = 0;
 	size=lseek(f, 0, SEEK_END);
 	fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0);
 	if (fdat==MAP_FAILED) {
@@ -95,6 +188,17 @@ int handleFile(int f, char *name, int compression, int level) {
 		return 0;
 	}
 
+#ifdef ESPFS_GZIP
+	if (shouldCompressGzip(name)) {
+		csize = size*3;
+		if (csize<100) // gzip has some headers that do not fit when trying to compress small files
+			csize = 100; // enlarge buffer if this is the case
+		cdat=malloc(csize);
+		csize=compressGzip(fdat, size, cdat, csize, level);
+		compression = COMPRESS_NONE;
+		flags = FLAG_GZIP;
+	} else
+#endif
 	if (compression==COMPRESS_NONE) {
 		csize=size;
 		cdat=fdat;
@@ -113,20 +217,25 @@ int handleFile(int f, char *name, int compression, int level) {
 		compression=COMPRESS_NONE;
 		csize=size;
 		cdat=fdat;
+		flags=0;
 	}
 
 	//Fill header data
 	h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24);
-	h.flags=0;
+	h.flags=flags;
 	h.compression=compression;
-	nameLen=strlen(name)+1;
+	h.nameLen=nameLen=strlen(name)+1;
-	if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary
+	if (h.nameLen&3) h.nameLen+=4-(h.nameLen&3); //Round to next 32bit boundary
-	h.nameLen=htoxs(nameLen);
+	h.nameLen=htoxs(h.nameLen);
 	h.fileLenComp=htoxl(csize);
 	h.fileLenDecomp=htoxl(size);
 	
 	write(1, &h, sizeof(EspFsHeader));
-	write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer.
+	write(1, name, nameLen);
+	while (nameLen&3) {
+		write(1, "\000", 1);
+		nameLen++;
+	}
 	write(1, cdat, csize);
 	//Pad out to 32bit boundary
 	while (csize&3) {
@@ -134,6 +243,20 @@ int handleFile(int f, char *name, int compression, int level) {
 		csize++;
 	}
 	munmap(fdat, size);
+
+	if (compName != NULL) {
+		if (h.compression==COMPRESS_HEATSHRINK) {
+			*compName = "heatshrink";
+		} else if (h.compression==COMPRESS_NONE) {
+			if (h.flags & FLAG_GZIP) {
+				*compName = "gzip";
+			} else {
+				*compName = "none";
+			}
+		} else {
+			*compName = "unknown";
+		}
+	}
 	return (csize*100)/size;
 }
 
@@ -149,7 +272,6 @@ void finishArchive() {
 	write(1, &h, sizeof(EspFsHeader));
 }
 
-
 int main(int argc, char **argv) {
 	int f, x;
 	char fileName[1024];
@@ -175,14 +297,29 @@ int main(int argc, char **argv) {
 			compLvl=atoi(argv[x+1]);
 			if (compLvl<1 || compLvl>9) err=1;
 			x++;
+#ifdef ESPFS_GZIP
+		} else if (strcmp(argv[x], "-g")==0 && argc>=x-2) {
+			if (!parseGzipExtensions(argv[x+1])) err=1;
+			x++;
+#endif
 		} else {
 			err=1;
 		}
 	}
 
+#ifdef ESPFS_GZIP
+	if (gzipExtensions == NULL) {
+		parseGzipExtensions(strdup("html,css,js"));
+	}
+#endif
+
 	if (err) {
 		fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
-		fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] > out.espfs\n", argv[0]);
+		fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] ", argv[0]);
+#ifdef ESPFS_GZIP
+		fprintf(stderr, "[-g gzipped_extensions] ");
+#endif
+		fprintf(stderr, "> out.espfs\n");
 		fprintf(stderr, "Compressors:\n");
 #ifdef ESPFS_HEATSHRINK
 		fprintf(stderr, "0 - None\n1 - Heatshrink(default)\n");
@@ -190,6 +327,9 @@ int main(int argc, char **argv) {
 		fprintf(stderr, "0 - None(default)\n");
 #endif
 		fprintf(stderr, "\nCompression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n");
+#ifdef ESPFS_GZIP
+		fprintf(stderr, "\nGzipped extensions: list of comma separated, case sensitive file extensions \nthat will be gzipped. Defaults to 'html,css,js'\n");
+#endif
 		exit(0);
 	}
 
@@ -205,8 +345,9 @@ int main(int argc, char **argv) {
 			if (realName[0]=='/') realName++;
 			f=open(fileName, O_RDONLY);
 			if (f>0) {
-				rate=handleFile(f, realName, compType, compLvl);
+				char *compName = "unknown";
-				fprintf(stderr, "%s (%d%%)\n", realName, rate);
+				rate=handleFile(f, realName, compType, compLvl, &compName);
+				fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName);
 				close(f);
 			} else {
 				perror(fileName);