Added heatshrink encoding to mkespfsimage. No decoding routines yet.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "lib/heatshrink"]
+	path = lib/heatshrink
+	url = https://github.com/atomicobject/heatshrink.git

Makefile

@@ -22,7 +22,7 @@ ESPTOOL		?= esptool.py
 ESPPORT		?= /dev/ttyUSB0
 
 # name for the target project
-TARGET		= ting
+TARGET		= httpd
 
 # which modules (subdirectories) of the project to include in compiling
 MODULES		= driver user
@@ -141,7 +141,8 @@ mkespfsimage/mkespfsimage: mkespfsimage/
 	make -C mkespfsimage
 
 htmlflash: webpages.espfs
-	-$(ESPTOOL) --port $(ESPPORT) write_flash 0x20000 webpages.espfs
+	if [ $$(stat -c '%s' webpages.espfs) -gt $$(( 0x30000 )) ]; then echo "webpages.espfs too big!"; false; fi
+	-$(ESPTOOL) --port $(ESPPORT) write_flash 0x10000 webpages.espfs
 
 clean:
 	$(Q) rm -f $(APP_AR)

mkespfsimage/Makefile

@@ -1,4 +1,5 @@
 
+CFLAGS=-I../lib/heatshrink -std=gnu99
 
-mkespfsimage: main.o
+mkespfsimage: main.o heatshrink_encoder.o
-	$(CC) -o mkespfsimage $<
+	$(CC) -o $@ $^

mkespfsimage/espfsformat.h

@@ -17,7 +17,7 @@ with the FLAG_LASTFILE flag set.
 
 #define FLAG_LASTFILE (1<<0)
 #define COMPRESS_NONE 0
-#define COMPRESS_LZF 1
+#define COMPRESS_HEATSHRINK 1
 
 typedef struct {
 	int32_t magic;

mkespfsimage/heatshrink_encoder.c

@@ -0,0 +1,3 @@
+//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir
+
+#include "../lib/heatshrink/heatshrink_encoder.c"

mkespfsimage/main.c

@@ -1,5 +1,5 @@
-
 #include <stdint.h>
+#include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -10,6 +10,12 @@
 #include <string.h>
 #include "espfsformat.h"
 
+//Heatshrink
+#include "heatshrink_common.h"
+#include "heatshrink_config.h"
+#include "heatshrink_encoder.h"
+
+
 //Routines to convert host format to the endianness used in the xtensa
 short htoxs(short in) {
 	char r[2];
@@ -27,38 +33,95 @@ int htoxl(int in) {
 	return *((int *)r);
 }
 
+size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) {
+	char *inp=in;
+	char *outp=out;
+	int len;
+	int ws[]={13, 11, 8, 6, 5};
+	int ls[]={4, 4, 4, 3, 3};
+	HSE_poll_res pres;
+	HSE_sink_res sres;
+	size_t r=0;
+	if (level==-1) level=8;
+	level=(level-1)/2; //level is now 0, 1, 2, 3, 4
+	heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]);
+	if (enc==NULL) {
+		perror("allocating mem for heatshrink");
+		exit(1);
+	}
+	do {
+		if (insize>0) {
+			sres=heatshrink_encoder_sink(enc, inp, insize, &len);
+			if (sres!=HSER_SINK_OK) break;
+			inp+=len; insize-=len;
+			if (insize==0) heatshrink_encoder_finish(enc);
+		}
+		do {
+			pres=heatshrink_encoder_poll(enc, outp, outsize, &len);
+			if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break;
+			outp+=len; outsize-=len;
+			r+=len;
+		} while (pres==HSER_POLL_MORE);
+	} while (insize!=0);
 
-void handleFile(int f, char *name) {
+	if (insize!=0) {
-	char *fdat;
+		fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres);
-	off_t size;
+		exit(1);
+	}
+
+	heatshrink_encoder_free(enc);
+	return r;
+}
+
+int handleFile(int f, char *name, int compression, int level) {
+	char *fdat, *cdat;
+	off_t size, csize;
 	EspFsHeader h;
 	int nameLen;
 	size=lseek(f, 0, SEEK_END);
 	fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0);
 	if (fdat==MAP_FAILED) {
 		perror("mmap");
-		return;
+		return 0;
+	}
+	
+	if (compression==COMPRESS_NONE) {
+		csize=size;
+		cdat=fdat;
+	} else if (compression==COMPRESS_HEATSHRINK) {
+		cdat=malloc(size*2);
+		csize=compressHeatshrink(fdat, size, cdat, size*2, level);
+	} else {
+		fprintf(stderr, "Unknown compression - %d\n", compression);
+		exit(1);
+	}
+
+	if (csize>size) {
+		//Compressing enbiggened this file. Revert to uncompressed store.
+		csize=size;
+		cdat=fdat;
 	}
 
 	//Fill header data
 	h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24);
 	h.flags=0;
-	h.compression=COMPRESS_NONE;
+	h.compression=compression;
 	nameLen=strlen(name)+1;
 	if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary
 	h.nameLen=htoxs(nameLen);
-	h.fileLenComp=htoxl(size);
+	h.fileLenComp=htoxl(csize);
 	h.fileLenDecomp=htoxl(size);
 	
 	write(1, &h, sizeof(EspFsHeader));
 	write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer.
-	write(1, fdat, size);
+	write(1, cdat, csize);
 	//Pad out to 32bit boundary
 	while (size&3) {
 		write(1, "\000", 1);
 		size++;
 	}
 	munmap(fdat, size);
+	return (csize*100)/size;
 }
 
 //Write final dummy header with FLAG_LASTFILE set.
@@ -75,11 +138,37 @@ void finishArchive() {
 
 
 int main(int argc, char **argv) {
-	int f;
+	int f, x;
 	char fileName[1024];
 	char *realName;
 	struct stat statBuf;
 	int serr;
+	int rate;
+	int err=0;
+	int compType=1; //default compression type - heatshrink
+	int compLvl=-1;
+
+	for (x=1; x<argc; x++) {
+		if (strcmp(argv[x], "-c")==0 && argc>=x-2) {
+			compType=atoi(argv[x=1]);
+			x++;
+		} else if (strcmp(argv[x], "-l")==0 && argc>=x-2) {
+			compLvl=atoi(argv[x=1]);
+			if (compLvl<1 || compLvl>9) err=1;
+			x++;
+		} else {
+			err=1;
+		}
+	}
+
+	if (err) {
+		fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
+		fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] > out.espfs\n", argv[0]);
+		fprintf(stderr, "Compressors:\n0 - None\n1 - Heatshrink(defautl\n");
+		fprintf(stderr, "Compression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n");
+		exit(0);
+	}
+
 	while(fgets(fileName, sizeof(fileName), stdin)) {
 		//Kill off '\n' at the end
 		fileName[strlen(fileName)-1]=0;
@@ -92,8 +181,8 @@ int main(int argc, char **argv) {
 			if (realName[0]=='/') realName++;
 			f=open(fileName, O_RDONLY);
 			if (f>0) {
-				handleFile(f, realName);
+				rate=handleFile(f, realName, compType, compLvl);
-				fprintf(stderr, "%s\n", realName);
+				fprintf(stderr, "%s (%d%%)\n", realName, rate);
 				close(f);
 			} else {
 				perror(fileName);
@@ -105,5 +194,6 @@ int main(int argc, char **argv) {
 		}
 	}
 	finishArchive();
+	return 0;
 }