code cleaning, add extract option to fs builder, fixes

demo/Makefile

@@ -22,7 +22,7 @@ staticfiles-embed.c: staticfiles.bin
 
 clean:
 	rm -f demo staticfiles.bin staticfiles-embed.c
-	make -C ../spritehttpd
+	make -C ../spritehttpd clean
 
 $(LIBFILE):
 	make -C ../spritehttpd PLATFORM=posix CFLAGS="-Og -g"

espfsbuilder/Makefile

@@ -13,7 +13,8 @@ CFLAGS = -I. \
 all: $(TARGET)
 
 $(TARGET): ${SOURCES}
-	cc -O3 -lz $^ -o $@ ${CFLAGS}
+	#cc -O3 -lz $^ -o $@ ${CFLAGS}
+	cc -Og -g -Wall -Wextra -lz $^ -o $@ ${CFLAGS}
 
 clean:
 	rm $(TARGET)

espfsbuilder/main.c

@@ -62,8 +62,8 @@ size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t
     size_t len;
     int ws[] = {5, 6, 8, 11, 13};
     int ls[] = {3, 3, 4, 4, 4};
-    HSE_poll_res pres;
-    HSE_sink_res sres;
+    HSE_poll_res pres = 0;
+    HSE_sink_res sres = 0;
     size_t r;
     if (level == -1) { level = 8; }
     level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4
@@ -358,7 +358,6 @@ int main(int argc, char **argv)
 {
     int f;
     char inputFileName[1024];
-    char *realName;
     struct stat statBuf;
     int serr;
     int rate;
@@ -373,11 +372,13 @@ int main(int argc, char **argv)
     char *outfile = NULL;
     char *parseFile = NULL;
     char *stripPath = NULL;
+    char *extractFile = NULL;
 
     while (1) {
         int option_index = 0;
         static struct option long_options[] = {
                 {"parse",      required_argument, 0, 'p'},
+                {"extract",    required_argument, 0, 'e'},
                 {"compress",   required_argument, 0, 'c'},
                 {"gzip",       no_argument,       0, 'z'},
                 {"gzip-all",   no_argument,       0, 'G'},
@@ -390,7 +391,7 @@ int main(int argc, char **argv)
                 {0, 0,                            0, 0}
         };
 
-        c = getopt_long(argc, argv, "c:l:g:zGhp:i:o:0123456789",
+        c = getopt_long(argc, argv, "c:l:g:zGS:e:hp:i:o:0123456789",
                         long_options, &option_index);
         if (c == -1) {
             break;
@@ -412,6 +413,10 @@ int main(int argc, char **argv)
                 parseFile = strdup(optarg);
                 break;
 
+            case 'e':
+                extractFile = strdup(optarg);
+                break;
+
             case 'S':
                 stripPath = strdup(optarg);
                 break;
@@ -457,8 +462,22 @@ int main(int argc, char **argv)
         }
     }
 
+    FILE *outfp = NULL;
+    if (outfile) {
+        fprintf(stderr, "Writing to %s\n", outfile);
+        outfp = fopen(outfile, "w+");
+        if (!outfp) {
+            perror(outfile);
+            return 1;
+        }
+        s_outFd = fileno(outfp);
+        ftruncate(s_outFd, 0);
+    } else {
+        fprintf(stderr, "Writing to stdout\n\n");
+    }
+
     if (parseFile) {
-        parseEspfsFileAndShowItsContents(parseFile);
+        parseEspfsImage(parseFile, extractFile, s_outFd);
         exit(0);
     }
 
@@ -481,20 +500,6 @@ int main(int argc, char **argv)
         }
     }
 
-    FILE *outfp = NULL;
-    if (outfile) {
-        fprintf(stderr, "Writing to %s\n", outfile);
-        outfp = fopen(outfile, "w+");
-        if (!outfp) {
-            perror(outfile);
-            return 1;
-        }
-        s_outFd = fileno(outfp);
-        ftruncate(s_outFd, 0);
-    } else {
-        fprintf(stderr, "Writing to stdout\n\n");
-    }
-
     struct InputFileLinkedListEntry *entry = s_inputFiles;
     while (entry) {
         char *name = entry->name;
@@ -545,6 +550,8 @@ int main(int argc, char **argv)
     fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
     fprintf(stderr, "Options:\n");
     fprintf(stderr, "[-p|--parse FILE]\n    Parse an espfs file and show a list of its contents. No other options apply in this mode.\n");
+    fprintf(stderr, "[-e|--extract FILE]\n    Extract a file with the given name from the parsed file (-p)\n");
+    fprintf(stderr, "[-S|--strip-path PATH]\n    Remove the given path from input file names before packing\n");
     fprintf(stderr, "[-c|--compress COMPRESSOR]\n    0 - None, 1 - Heatshrink (default)\n");
     fprintf(stderr, "[-l|--level LEVEL] or [-0 through -9]\n    compression level 1-9, higher is better but uses more RAM\n");
     fprintf(stderr, "[-z|--gzip]\n    use gzip for files with extensions matching "DEFAULT_GZIP_EXTS"\n");

espfsbuilder/parsing.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <zlib.h>
 
 #include "parsing.h"
 #include "espfs.h"
@@ -9,20 +10,78 @@
 static size_t espfs_parse_filesize = -1;
 static int espfs_parse_fd = -1;
 
+size_t decompressGzip(const uint8_t *in, size_t insize, int outfd)
+{
+#define OUTBUF_LEN 10240
+    uint8_t outbuf[OUTBUF_LEN];
+
+    z_stream stream;
+    int zresult;
+
+    stream.zalloc = Z_NULL;
+    stream.zfree = Z_NULL;
+    stream.opaque = Z_NULL;
+    stream.next_in = in;
+    stream.avail_in = insize;
+    stream.next_out = outbuf;
+    stream.avail_out = OUTBUF_LEN;
+    // 31 -> 15 window bits + 16 for gzip
+    zresult = inflateInit2(&stream, 31);
+    if (zresult != Z_OK) {
+        fprintf(stderr, "InflateInit2 failed with code %d\n", zresult);
+        exit(1);
+    }
+
+    while (1) {
+        stream.avail_out = OUTBUF_LEN;
+        stream.next_out = outbuf;
+
+        fprintf(stderr, "inflate chunk\n");
+
+        zresult = inflate(&stream, Z_FINISH);
+        if (zresult == Z_BUF_ERROR || zresult == Z_OK || zresult == Z_STREAM_END) {
+            int have = OUTBUF_LEN - stream.avail_out;
+            fprintf(stderr, "inflated: %d\n", have);
+            if (have != write(outfd, outbuf, have)) {
+                perror("Write output");
+                exit(1);
+            }
+            if (zresult == Z_STREAM_END) {
+                fprintf(stderr, "Z_STREAM_END\n");
+                break;
+            }
+        } else {
+            fprintf(stderr, "gzip error: %d\n", zresult);
+            exit(1);
+        }
+    }
+
+    zresult = inflateEnd(&stream);
+    if (zresult != Z_OK) {
+        fprintf(stderr, "InflateEnd failed with code %d\n", zresult);
+        exit(1);
+    }
+
+    fprintf(stderr, "Total decoded = %d\n", (int) stream.total_out);
+    return stream.total_out;
+}
+
 /**
  * Parse an image file and show the files contained.
  * This is a simple sanity test.
  *
- * @param[in] filename - image file to parse
+ * @param[in] imagefile - image file to parse
+ * @param[in] extractfile - name of the file to extract
+ * @param outfd - output FD when extracting
  */
-void parseEspfsFileAndShowItsContents(const char *filename)
+void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
 {
     int rv;
-    fprintf(stderr, "Parsing: %s\n", filename);
+    fprintf(stderr, "Parsing: %s\n", imagefile);
 
-    FILE *f = fopen(filename, "r");
+    FILE *f = fopen(imagefile, "r");
     if (!f) {
-        perror(filename);
+        perror(imagefile);
         exit(1);
     }
     int fd = fileno(f);
@@ -38,6 +97,45 @@ void parseEspfsFileAndShowItsContents(const char *filename)
         exit(1);
     }
 
+    if (extractfile) {
+        EspFsFile * efile = espFsOpen(extractfile);
+        EspFsHeader hdr;
+
+        if (!efile) {
+            fprintf(stderr, "Fail to open file %s from image\n", extractfile);
+            exit(1);
+        }
+
+        rv = espFsFileReadHeader(efile, &hdr);
+        if (rv != 0) {
+            fprintf(stderr, "Fail to read file header\n");
+            exit(1);
+        }
+
+        bool isGzip = hdr.flags & FLAG_GZIP;
+
+        size_t expected_readlen = isGzip ? hdr.fileLenComp : hdr.fileLenDecomp;
+
+        uint8_t *buff = malloc(expected_readlen);
+        int lenRead = espFsRead(efile, buff, expected_readlen);
+        if (lenRead != (int) expected_readlen) {
+            fprintf(stderr, "Fail to read raw file from espfs image - read len %d", lenRead);
+            exit(1);
+        }
+
+        if (isGzip) {
+            fprintf(stderr, "[EspFS] File is gzipped!");
+            decompressGzip(buff, lenRead, outfd);
+        } else {
+            write(outfd, buff, lenRead);
+        }
+
+        fsync(outfd);
+        exit(0);
+    }
+
+    /* Walk the image */
+
     EspFsWalk walk;
     espFsWalkInit(&walk);
 
@@ -58,7 +156,7 @@ int httpdPlatEspfsRead(void *dest, uint32_t offset, size_t len)
 {
     // fprintf(stderr, "FS read @ %d, len %d\n", offset, (int) len);
     if (offset + len > espfs_parse_filesize) {
-        // fprintf(stderr, "Read out fo range!\n");
+        fprintf(stderr, "Read out fo range!\n");
         return -1;
     }
     lseek(espfs_parse_fd, offset, SEEK_SET);

espfsbuilder/parsing.h

@@ -4,4 +4,4 @@
 
 #pragma once
 
-void parseEspfsFileAndShowItsContents(const char *filename);
+void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd);

espfsbuilder/testfiles/LIMECURD.TXT

@@ -0,0 +1,7 @@
+Fruit curd is a dessert spread and topping usually made with citrus fruit, such as lemon, lime, orange, or tangerine. Other flavor variations include passion fruit, mango, and berries such as raspberries, cranberries or blackberries. The basic ingredients are beaten egg yolks, sugar, fruit juice, and zest, which are gently cooked together until thick and then allowed to cool, forming a soft, smooth, intensely flavoured spread. Some recipes also include egg whites and/or butter.
+
+In late 19th and early 20th century England, home-made lemon curd was traditionally served with bread or scones at afternoon tea as an alternative to jam, and as a filling for cakes, small pastries, and tarts. Homemade lemon curd was usually made in relatively small amounts as it did not keep as well as jam. In more modern times, larger quantities became possible because of the use of refrigeration. Commercially manufactured curds often contain additional preservatives and thickening agents.
+
+Contemporary commercially made curds remain a popular spread for bread, scones, toast, waffles, crumpets, pancakes, cheesecake, or muffins. They can also be used as a flavoring for desserts or yoghurt. Lemon-meringue pie, made with lemon curd and topped with meringue, has been a popular dessert in Britain and the United States since the nineteenth century. Lemon curd can also have whipped cream folded into it for such uses as filling cream puffs.
+
+Curds differ from pie fillings or custards in that they contain a higher proportion of juice and zest, which gives them a more intense flavor. Also, curds containing butter have a smoother and creamier texture than both pie fillings and custards, which contain little or no butter and use cornstarch or flour for thickening. Additionally, unlike custards, curds are not usually eaten on their own.

espfsbuilder/testout/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

spritehttpd/Makefile

@@ -30,7 +30,8 @@ LIB_OBJS = ${LIB_SOURCES:.c=.o}
 LIB_INCLUDES = -Iinclude -Ilib/heatshrink -Ilib/espfs
 
 # TODO check what these mean
-LIB_CFLAGS = -fPIC -Wall -Wextra -c
+#LIB_CFLAGS = -fPIC -Wall -Wextra -c
+LIB_CFLAGS = -fPIC -Wall -Wextra -c -Og -g
 
 OBJ_DIR=./obj
 

spritehttpd/include/cgiwebsocket.h

@@ -11,7 +11,7 @@ typedef struct Websock Websock;
 typedef struct WebsockPriv WebsockPriv;
 
 typedef void(*WsConnectedCb)(Websock *ws);
-typedef void(*WsRecvCb)(Websock *ws, char *data, int len, int flags);
+typedef void(*WsRecvCb)(Websock *ws, uint8_t *data, size_t len, int flags);
 typedef void(*WsSentCb)(Websock *ws);
 typedef void(*WsCloseCb)(Websock *ws);
 
@@ -26,8 +26,8 @@ struct Websock {
 };
 
 httpd_cgi_state cgiWebsocket(HttpdConnData *connData);
-int cgiWebsocketSend(Websock *ws, const char *data, int len, int flags);
+int cgiWebsocketSend(Websock *ws, const uint8_t *data, size_t len, int flags);
 void cgiWebsocketClose(Websock *ws, int reason);
-httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, char *data, int len);
-int cgiWebsockBroadcast(const char *resource, const char *data, int len, int flags);
-void cgiWebsockMeasureBacklog(const char *resource, int *total, int *max);
+httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, uint8_t *data, size_t len);
+int cgiWebsockBroadcast(const char *resource, const uint8_t *data, size_t len, int flags);
+void cgiWebsockMeasureBacklog(const char *resource, size_t *total, size_t *max);

spritehttpd/include/httpd.h

@@ -2,6 +2,8 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <stddef.h>
+#include <sys/types.h> /* needed for ssize_t */
 #include "httpd-platform.h"
 
 #ifndef GIT_HASH
@@ -87,7 +89,7 @@ typedef struct HttpdPostData HttpdPostData;
 extern HttpdConnData *s_connData[HTTPD_MAX_CONNECTIONS];
 
 typedef httpd_cgi_state (* cgiSendCallback)(HttpdConnData *connData);
-typedef httpd_cgi_state (* cgiRecvHandler)(HttpdConnData *connData, char *data, int len);
+typedef httpd_cgi_state (* cgiRecvHandler)(HttpdConnData *connData, char *data, size_t len);
 
 struct httpd_options {
   uint16_t port;
@@ -184,16 +186,53 @@ void httpdStartResponse(HttpdConnData *conn, int code);
 void httpdHeader(HttpdConnData *conn, const char *field, const char *val);
 void httpdEndHeaders(HttpdConnData *conn);
 int httpdGetHeader(HttpdConnData *conn, const char *header, char *ret, int retLen);
-int httpdSend(HttpdConnData *conn, const char *data, int len);
-int httpdSend_js(HttpdConnData *conn, const char *data, int len);
-int httpdSend_html(HttpdConnData *conn, const char *data, int len);
+
+/**
+ * Send binary data
+ *
+ * @param conn
+ * @param data - data to send
+ * @param len - num bytes. -1 to use strlen.
+ * @return 1 = success
+ */
+int httpdSend(HttpdConnData *conn, const uint8_t *data, size_t len);
+
+/**
+ * Send a string. The length is measured using strlen.
+ *
+ * @param conn
+ * @param data - string
+ * @return 1 = success
+ */
+static inline int httpdSendStr(HttpdConnData *conn, const char *data)
+{
+    return httpdSend(conn, (const uint8_t *) data, strlen(data));
+}
+
+/**
+ * Send a string with custom length. This is a slight optimization over httpdSendStr when the length is known.
+ *
+ * @param conn
+ * @param data - string
+ * @param len - num bytes
+ * @return 1 = success
+ */
+static inline int httpdSendStrN(HttpdConnData *conn, const char *data, size_t len)
+{
+    return httpdSend(conn, (const uint8_t *) data, len);
+}
+
+// TODO convert to a general escaped send function
+int httpdSend_js(HttpdConnData *conn, const uint8_t *data, ssize_t len);
+int httpdSend_html(HttpdConnData *conn, const uint8_t *data, ssize_t len);
+
 bool httpdFlushSendBuffer(HttpdConnData *conn);
 void httpdContinue(HttpdConnData *conn);
 void httpdConnSendStart(HttpdConnData *conn);
 void httpdConnSendFinish(HttpdConnData *conn);
 void httpdAddCacheHeaders(HttpdConnData *connData, const char *mime);
 
-int httpGetBacklogSize(const HttpdConnData *connData);
+size_t httpGetBacklogSize(const HttpdConnData *connData);
 void httdResponseOptions(HttpdConnData *conn, int cors);
 
 //Platform dependent code should call these.

spritehttpd/lib/espfs/espfs.c

@@ -22,21 +22,23 @@ It's written for use with httpd, but doesn't need to be used as such.
 #include "espfsformat.h"
 #include "espfs.h"
 
-#include "heatshrink_decoder.h"
 #include "logging.h"
 
-// forward declaration for use in the stand-alone espfs tool
-int httpdPlatEspfsRead(void *dest, uint32_t offset, size_t len);
-
+// internal fields
 struct EspFsFile {
-  uint32_t header;
-  char decompressor;
+  /// Header pointer
+  uint32_t headerPos;
+  /// Decompressor type
+  uint8_t decompressor;
   uint32_t posDecomp;
   uint32_t posStart;
   uint32_t posComp;
   heatshrink_decoder *decompData;
 };
 
+// forward declaration for use in the stand-alone espfs tool
+int httpdPlatEspfsRead(void *dest, uint32_t offset, size_t len);
+
 
 EspFsInitResult espFsInit()
 {
@@ -60,7 +62,7 @@ int espFsFlags(EspFsFile *fh)
     }
 
     int8_t flags;
-    httpdPlatEspfsRead(&flags, fh->header + offsetof(EspFsHeader, flags), 1);
+    httpdPlatEspfsRead(&flags, fh->headerPos + offsetof(EspFsHeader, flags), 1);
     return (int) flags;
 }
 
@@ -133,13 +135,13 @@ EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos)
         return NULL;
     }
 
-    hpos += sizeof(EspFsHeader);
-    hpos += h->nameLen; // Skip to content
-
     EspFsFile *r = (EspFsFile *) httpdPlatMalloc(sizeof(EspFsFile)); //Alloc file desc mem
     if (r == NULL) { return NULL; }
-    r->header = hpos;
+    r->headerPos = hpos;
     r->decompressor = h->compression;
+
+    hpos += sizeof(EspFsHeader);
+    hpos += h->nameLen; // Skip to content
     r->posComp = hpos;
     r->posStart = hpos;
     r->posDecomp = 0;
@@ -148,6 +150,7 @@ EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos)
 
     if (h->compression == COMPRESS_NONE) {
         r->decompData = NULL;
+        return r;
     } else if (h->compression == COMPRESS_HEATSHRINK) {
         //File is compressed with Heatshrink.
         char parm;
@@ -166,6 +169,7 @@ EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos)
         httpdPlatFree(r);
         return NULL;
     }
+    return NULL;
 }
 
 //Open a file and return a pointer to the file desc struct.
@@ -179,6 +183,10 @@ EspFsFile *espFsOpenAt(uint32_t hpos)
     return espFsOpenFromHeader(&h, hpos);
 }
 
+int espFsFileReadHeader(const EspFsFile *file, EspFsHeader *header)
+{
+    return httpdPlatEspfsRead(header, file->headerPos, sizeof(EspFsHeader));
+}
 
 //Open a file and return a pointer to the file desc struct.
 EspFsFile *espFsOpen(const char *fileName)
@@ -188,7 +196,6 @@ EspFsFile *espFsOpen(const char *fileName)
     uint32_t hpos;
     char namebuf[256];
     EspFsHeader h;
-    EspFsFile *r;
     //Strip initial slashes
     while (fileName[0] == '/') { fileName++; }
 
@@ -241,14 +248,16 @@ EspFsFile *espFsOpen(const char *fileName)
 }
 
 //Read len bytes from the given file into buff. Returns the actual amount of bytes read.
-int espFsRead(EspFsFile *fh, char *buff, size_t len)
+int espFsRead(EspFsFile *fh, uint8_t *buff, size_t buf_cap)
 {
-    int rv;
-    int flen;
-    int fdlen;
+    int rv = 0;
+    uint32_t binary_len = 0;
+    uint32_t decompressed_len = 0;
     if (fh == NULL) { return 0; }
 
-    rv = httpdPlatEspfsRead(&flen, fh->header + offsetof(EspFsHeader, fileLenComp), 4);
+    espfs_dbg("[EspFS] File read, pos @%d: cap %d", (int)fh->posComp, (int) buf_cap);
+
+    rv = httpdPlatEspfsRead(&binary_len, fh->headerPos + offsetof(EspFsHeader, fileLenComp), 4);
     if (rv != 0) {
         return 0;
     }
@@ -256,62 +265,68 @@ int espFsRead(EspFsFile *fh, char *buff, size_t len)
     //Cache file length.
     //Do stuff depending on the way the file is compressed.
     if (fh->decompressor == COMPRESS_NONE) {
-        int toRead = flen - (int) (fh->posComp - fh->posStart);
+        int toRead = (int) binary_len - (int) (fh->posComp - fh->posStart);
         if (toRead < 0) { toRead = 0; }
-        if (len > toRead) { len = toRead; }
-        rv = httpdPlatEspfsRead(buff, fh->posComp, len);
+        if ((int) buf_cap > toRead) { buf_cap = toRead; }
+        espfs_dbg("[EspFS] Plain data, read chunk @%d: %d", (int)fh->posComp, (int) buf_cap);
+        rv = httpdPlatEspfsRead(buff, fh->posComp, buf_cap);
         if (rv != 0) {
             return 0;
         }
 
-        fh->posDecomp += len;
-        fh->posComp += len;
-        return (int) len;
+        fh->posDecomp += buf_cap;
+        fh->posComp += buf_cap;
+        return (int) buf_cap;
     } else if (fh->decompressor == COMPRESS_HEATSHRINK) {
-        rv = httpdPlatEspfsRead(&fdlen, fh->header + offsetof(EspFsHeader, fileLenDecomp), 4);
+        rv = httpdPlatEspfsRead(&decompressed_len, fh->headerPos + offsetof(EspFsHeader, fileLenDecomp), 4);
         if (rv != 0) {
             return 0;
         }
 
-        size_t decoded = 0;
-        size_t elen, rlen;
-        char ebuff[16];
+        size_t decoded_bytes = 0;
+        size_t remaining_binary_len, sunk_len, decoded_chunk_len;
+#define HS_DECOMP_CHUNK_LEN 16
+        char ebuff[HS_DECOMP_CHUNK_LEN];
         heatshrink_decoder *dec = fh->decompData;
-        if (fh->posDecomp == fdlen) {
-            return 0;
-        }
 
         // We must ensure that whole file is decompressed and written to output buffer.
         // This means even when there is no input data (elen==0) try to poll decoder until
         // posDecomp equals decompressed file length
 
-        while (decoded < len) {
+        while (decoded_bytes < buf_cap) {
             //Feed data into the decompressor
             //ToDo: Check ret val of heatshrink fns for errors
-            elen = flen - (fh->posComp - fh->posStart);
-            if (elen > 0) {
-                rv = httpdPlatEspfsRead(ebuff, fh->posComp, 16);
+            remaining_binary_len = binary_len - (fh->posComp - fh->posStart);
+            if (remaining_binary_len > 0) {
+                const size_t chunk = remaining_binary_len < HS_DECOMP_CHUNK_LEN ?
+                        remaining_binary_len : HS_DECOMP_CHUNK_LEN;
+
+                espfs_dbg("[EspFS] HS data, read chunk @%d: %d", (int)fh->posComp, (int) chunk);
+                rv = httpdPlatEspfsRead(ebuff, fh->posComp, chunk);
                 if (rv != 0) {
                     return 0;
                 }
 
-                heatshrink_decoder_sink(dec, (uint8_t *) ebuff, (elen > 16) ? 16 : elen, &rlen);
-                fh->posComp += rlen;
+                heatshrink_decoder_sink(dec, (uint8_t *) ebuff, chunk, &sunk_len);
+                espfs_dbg("[EspFS] HS sunk %d bytes", (int) sunk_len);
+                fh->posComp += sunk_len;
             }
             //Grab decompressed data and put into buff
-            heatshrink_decoder_poll(dec, (uint8_t *) buff, len - decoded, &rlen);
-            fh->posDecomp += rlen;
-            buff += rlen;
-            decoded += rlen;
-
-            if (elen == 0) {
-                if (fh->posDecomp == fdlen) {
+            decoded_chunk_len = 0;
+            heatshrink_decoder_poll(dec, buff, buf_cap - decoded_bytes, &decoded_chunk_len);
+            fh->posDecomp += decoded_chunk_len;
+            buff += decoded_chunk_len;
+            decoded_bytes += decoded_chunk_len;
+
+            espfs_dbg("[EspFS] HS pulled %d bytes, remaining binary len %d", (int) decoded_chunk_len, (int) remaining_binary_len);
+            if (remaining_binary_len == 0) {
+                if (fh->posDecomp == decompressed_len) {
                     heatshrink_decoder_finish(dec);
                 }
-                return (int) decoded;
+                return (int) decoded_bytes;
             }
         }
-        return (int) len;
+        return (int) buf_cap;
     }
     return 0;
 }

spritehttpd/lib/espfs/espfs.h

@@ -3,6 +3,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include "espfsformat.h"
+#include "heatshrink_decoder.h"
 
 typedef enum {
 	ESPFS_INIT_RESULT_OK,
@@ -17,6 +18,14 @@ struct EspFsWalk {
 };
 typedef struct EspFsWalk EspFsWalk;
 
+/**
+ * Read a file header
+ *
+ * @param[in] file - file to read
+ * @param[out] header - header is read here
+ * @return 0 = success
+ */
+int espFsFileReadHeader(const EspFsFile *file, EspFsHeader *header);
 /** Init filesystem walk */
 void espFsWalkInit(EspFsWalk *walk);
 /**
@@ -38,5 +47,5 @@ EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos);
 EspFsInitResult espFsInit();
 EspFsFile *espFsOpen(const char *fileName);
 int espFsFlags(EspFsFile *fh);
-int espFsRead(EspFsFile *fh, char *buff, size_t len);
+int espFsRead(EspFsFile *fh, uint8_t *buff, size_t len);
 void espFsClose(EspFsFile *fh);

spritehttpd/src/cgiwebsocket.c

@@ -88,7 +88,7 @@ struct WebsockPriv {
 
 static Websock *llStart = NULL;
 
-static int sendFrameHead(Websock *ws, int opcode, int len)
+static int sendFrameHead(Websock *ws, int opcode, size_t len)
 {
     uint8_t buf[14];
     int i = 0;
@@ -111,10 +111,10 @@ static int sendFrameHead(Websock *ws, int opcode, int len)
         buf[i++] = len;
     }
 //	ws_dbg("WS: Sent frame head for payload of %d bytes.", len);
-    return httpdSend(ws->conn, (char *) buf, i);
+    return httpdSend(ws->conn, buf, i);
 }
 
-int cgiWebsocketSend(Websock *ws, const char *data, int len, int flags)
+int cgiWebsocketSend(Websock *ws, const uint8_t *data, size_t len, int flags)
 {
     int r = 0;
     int fl = 0;
@@ -135,7 +135,7 @@ int cgiWebsocketSend(Websock *ws, const char *data, int len, int flags)
 }
 
 //Broadcast data to all websockets at a specific url. Returns the amount of connections sent to.
-int cgiWebsockBroadcast(const char *resource, const char *data, int len, int flags)
+int cgiWebsockBroadcast(const char *resource, const uint8_t *data, size_t len, int flags)
 {
     Websock *lw = llStart;
     int ret = 0;
@@ -166,15 +166,15 @@ int cgiWebsockBroadcast(const char *resource, const char *data, int len, int fla
 }
 
 /** this is used for estimation how full the ram is */
-void cgiWebsockMeasureBacklog(const char *resource, int *total, int *max)
+void cgiWebsockMeasureBacklog(const char *resource, size_t *total, size_t *max)
 {
     Websock *lw = llStart;
-    int bMax = 0;
-    int bTotal = 0;
+    size_t bMax = 0;
+    size_t bTotal = 0;
     while (lw != NULL) {
         if (strcmp(lw->conn->url, resource) == 0) {
             //lw->conn
-            int bs = httpGetBacklogSize(lw->conn);
+            size_t bs = httpGetBacklogSize(lw->conn);
             bTotal += bs;
             if (bs > bMax) { bMax = bs; }
         }
@@ -186,7 +186,7 @@ void cgiWebsockMeasureBacklog(const char *resource, int *total, int *max)
 
 void cgiWebsocketClose(Websock *ws, int reason)
 {
-    char rs[2] = {reason >> 8, reason & 0xff};
+    uint8_t rs[2] = {reason >> 8, reason & 0xff};
     sendFrameHead(ws, FLAG_FIN | OPCODE_CLOSE, 2);
     httpdSend(ws->conn, rs, 2);
     ws->priv->closedHere = 1;
@@ -210,13 +210,13 @@ static void websockFree(Websock *ws)
     if (ws->priv) { httpdPlatFree(ws->priv); }
 }
 
-httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, char *data, int len)
+httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, uint8_t *data, size_t len)
 {
-    int i, j, sl;
+    int j, sl;
     httpd_cgi_state r = HTTPD_CGI_MORE;
     int wasHeaderByte;
     Websock *ws = (Websock *) connData->cgiData;
-    for (i = 0; i < len; i++) {
+    for (size_t i = 0; i < len; i++) {
 //		httpd_printf("Ws: State %d byte 0x%02X\n", ws->priv->wsStatus, data[i]);
         wasHeaderByte = 1;
         if (ws->priv->wsStatus == ST_FLAGS) {
@@ -261,7 +261,7 @@ httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, char *data, int len)
             //First, unmask the data
             sl = len - i;
 //			ws_dbg("Ws: Frame payload. wasHeaderByte %d fr.len %d sl %d cmd 0x%x", wasHeaderByte, (int)ws->priv->fr.len, (int)sl, ws->priv->fr.flags);
-            if (sl > ws->priv->fr.len) { sl = ws->priv->fr.len; }
+            if ((uint64_t) sl > ws->priv->fr.len) { sl = (int) ws->priv->fr.len; }
             for (j = 0; j < sl; j++) { data[i + j] ^= (ws->priv->fr.mask[(ws->priv->maskCtr++) & 3]); }
 
 //			if (DEBUG_WS) {
@@ -283,7 +283,7 @@ httpd_cgi_state cgiWebSocketRecv(HttpdConnData *connData, char *data, int len)
             } else if ((ws->priv->fr.flags & OPCODE_MASK) == OPCODE_TEXT ||
                        (ws->priv->fr.flags & OPCODE_MASK) == OPCODE_BINARY ||
                        (ws->priv->fr.flags & OPCODE_MASK) == OPCODE_CONTINUE) {
-                if (sl > ws->priv->fr.len) { sl = ws->priv->fr.len; }
+                if ((uint64_t) sl > ws->priv->fr.len) { sl = ws->priv->fr.len; }
                 if (!(ws->priv->fr.len8 & IS_MASKED)) {
                     //We're a server; client should send us masked packets.
                     cgiWebsocketClose(ws, 1002);

spritehttpd/src/httpd.c

@@ -162,7 +162,7 @@ const char *httpdGetVersion(void)
     return HTTPDVER;
 }
 
-int httpGetBacklogSize(const HttpdConnData *conn)
+size_t httpGetBacklogSize(const HttpdConnData *conn)
 {
     HttpSendBacklogItem *bl = conn->priv->sendBacklog;
     if (!bl) { return 0; }
@@ -193,6 +193,9 @@ static HttpdConnData *httpdFindConnData(ConnTypePtr conn, const char *remIp, int
 //Retires a connection for re-use
 static void httpdRetireConn(HttpdConnData *conn)
 {
+    if (!conn) {
+        return;
+    }
     if (conn->priv->sendBacklog != NULL) {
         HttpSendBacklogItem *i, *j;
         i = conn->priv->sendBacklog;
@@ -205,14 +208,14 @@ static void httpdRetireConn(HttpdConnData *conn)
     if (conn->post->buff != NULL) { httpdPlatFree(conn->post->buff); }
     if (conn->post != NULL) { httpdPlatFree(conn->post); }
     if (conn->priv != NULL) { httpdPlatFree(conn->priv); }
-    if (conn) { httpdPlatFree(conn); }
+    httpdPlatFree(conn);
     for (int i = 0; i < HTTPD_MAX_CONNECTIONS; i++) {
         if (s_connData[i] == conn) { s_connData[i] = NULL; }
     }
 }
 
 //Stupid li'l helper function that returns the value of a hex char.
-static int httpdHexVal(char c)
+static char httpdHexVal(char c)
 {
     if (c >= '0' && c <= '9') { return c - '0'; }
     if (c >= 'A' && c <= 'F') { return c - 'A' + 10; }
@@ -227,13 +230,14 @@ static int httpdHexVal(char c)
 int httpdUrlDecode(const char *val, int valLen, char *ret, int retLen)
 {
     int s = 0, d = 0;
-    int esced = 0, escVal = 0;
+    int esced = 0;
+    char escVal = 0;
     while (s < valLen && d < retLen) {
         if (esced == 1) {
             escVal = httpdHexVal(val[s]) << 4;
             esced = 2;
         } else if (esced == 2) {
-            escVal += httpdHexVal(val[s]);
+            escVal |= httpdHexVal(val[s]);
             ret[d++] = escVal;
             esced = 0;
         } else if (val[s] == '%') {
@@ -266,8 +270,8 @@ int httpdFindArg(const char *line, const char *arg, char *buff, int buffLen)
             p += arglen + 1; //move p to start of value
             e = strstr(p, "&");
             if (e == NULL) { e = p + strlen(p); }
-                    router_dbg("findArg: val %s len %d", p, (e - p));
-            return httpdUrlDecode(p, (e - p), buff, buffLen);
+            router_dbg("findArg: val %s len %d", p, (int) (e - p));
+            return httpdUrlDecode(p, (int)(e - p), buff, buffLen);
         }
         p = strstr(p, "&");
         if (p != NULL) { p += 1; }
@@ -339,28 +343,28 @@ void httpdStartResponse(HttpdConnData *conn, int code)
                 code2str(code),
                 serverName,
                 connStr);
-    httpdSend(conn, buff, l);
+    httpdSendStrN(conn, buff, l);
 
     if (0 == (conn->priv->flags & HFL_NOCORS)) {
         // CORS headers
-        httpdSend(conn, "Access-Control-Allow-Origin: *\r\n", -1);
-        httpdSend(conn, "Access-Control-Allow-Methods: GET,POST,OPTIONS\r\n", -1);
+        httpdSendStr(conn, "Access-Control-Allow-Origin: *\r\n");
+        httpdSendStr(conn, "Access-Control-Allow-Methods: GET,POST,OPTIONS\r\n");
     }
 }
 
 //Send a http header.
 void httpdHeader(HttpdConnData *conn, const char *field, const char *val)
 {
-    httpdSend(conn, field, -1);
-    httpdSend(conn, ": ", -1);
-    httpdSend(conn, val, -1);
-    httpdSend(conn, "\r\n", -1);
+    httpdSendStr(conn, field);
+    httpdSendStr(conn, ": ");
+    httpdSendStr(conn, val);
+    httpdSendStr(conn, "\r\n");
 }
 
 //Finish the headers.
 void httpdEndHeaders(HttpdConnData *conn)
 {
-    httpdSend(conn, "\r\n", -1);
+    httpdSendStr(conn, "\r\n");
     conn->priv->flags |= HFL_SENDINGBODY;
 }
 
@@ -371,8 +375,8 @@ void httpdRedirect(HttpdConnData *conn, const char *newUrl)
     httpdStartResponse(conn, 302);
     httpdHeader(conn, "Location", newUrl);
     httpdEndHeaders(conn);
-    httpdSend(conn, "Moved to ", -1);
-    httpdSend(conn, newUrl, -1);
+    httpdSendStr(conn, "Moved to ");
+    httpdSendStr(conn, newUrl);
 }
 
 //Use this as a cgi function to redirect one url to another.
@@ -392,7 +396,7 @@ static httpd_cgi_state cgiNotFound(HttpdConnData *connData)
     if (connData->conn == NULL) { return HTTPD_CGI_DONE; }
     httpdStartResponse(connData, 404);
     httpdEndHeaders(connData);
-    httpdSend(connData, "404 File not found.", -1);
+    httpdSendStr(connData, "404 File not found.");
     return HTTPD_CGI_DONE;
 }
 
@@ -406,7 +410,6 @@ httpd_cgi_state cgiRedirectToHostname(HttpdConnData *connData)
     static const char hostFmt[] = "http://%s/";
     char *buff;
     int isIP = 0;
-    int x;
     if (connData->conn == NULL) {
         //Connection aborted. Clean up.
         return HTTPD_CGI_DONE;
@@ -419,7 +422,7 @@ httpd_cgi_state cgiRedirectToHostname(HttpdConnData *connData)
     //Quick and dirty code to see if host is an IP
     if (strlen(connData->hostName) > 8) {
         isIP = 1;
-        for (x = 0; x < strlen(connData->hostName); x++) {
+        for (size_t x = 0; x < strlen(connData->hostName); x++) {
             if (connData->hostName[x] != '.' && (connData->hostName[x] < '0' || connData->hostName[x] > '9')) { isIP = 0; }
         }
     }
@@ -443,10 +446,9 @@ httpd_cgi_state cgiRedirectToHostname(HttpdConnData *connData)
 //Add data to the send buffer. len is the length of the data. If len is -1
 //the data is seen as a C-string.
 //Returns 1 for success, 0 for out-of-memory.
-int httpdSend(HttpdConnData *conn, const char *data, int len)
+int httpdSend(HttpdConnData *conn, const uint8_t *data, size_t len)
 {
     if (conn->conn == NULL) { return 0; }
-    if (len < 0) { len = strlen(data); }
     if (len == 0) { return 0; }
     if (conn->priv->flags & HFL_CHUNKED && conn->priv->flags & HFL_SENDINGBODY && conn->priv->chunkHdr == NULL) {
         if (conn->priv->sendBuffLen + len + 6 > HTTPD_MAX_SENDBUFF_LEN) { return 0; }
@@ -464,19 +466,21 @@ int httpdSend(HttpdConnData *conn, const char *data, int len)
 static char httpdHexNibble(int val)
 {
     val &= 0xf;
-    if (val < 10) { return '0' + val; }
-    return 'A' + (val - 10);
+    if (val < 10) { return (char) ('0' + val); }
+    return (char) ('A' + (val - 10));
 }
 
-#define httpdSend_orDie(conn, data, len) do { if (!httpdSend((conn), (data), (len))) return false; } while (0)
+#define httpdSend_orDie(conn, data, len) do { if (!httpdSend((conn), (const uint8_t *)(data), (len))) return false; } while (0)
+
+#define httpdSendStr_orDie(conn, data) do { if (!httpdSendStr((conn), (data))) return false; } while (0)
 
 /* encode for HTML. returns 0 or 1 - 1 = success */
-int httpdSend_html(HttpdConnData *conn, const char *data, int len)
+int httpdSend_html(HttpdConnData *conn, const uint8_t *data, ssize_t len)
 {
     int start = 0, end = 0;
-    char c;
+    uint8_t c;
     if (conn->conn == NULL) { return 0; }
-    if (len < 0) { len = (int) strlen(data); }
+    if (len < 0) { len = (int) strlen((const char *) data); }
     if (len == 0) { return 0; }
 
     for (end = 0; end < len; end++) {
@@ -491,10 +495,10 @@ int httpdSend_html(HttpdConnData *conn, const char *data, int len)
             start = end + 1;
         }
 
-        if (c == '"') httpdSend_orDie(conn, "&#34;", 5);
-        else if (c == '\'') httpdSend_orDie(conn, "&#39;", 5);
-        else if (c == '<') httpdSend_orDie(conn, "&lt;", 4);
-        else if (c == '>') httpdSend_orDie(conn, "&gt;", 4);
+        if (c == '"') httpdSendStr_orDie(conn, "&#34;");
+        else if (c == '\'') httpdSendStr_orDie(conn, "&#39;");
+        else if (c == '<') httpdSendStr_orDie(conn, "&lt;");
+        else if (c == '>') httpdSendStr_orDie(conn, "&gt;");
     }
 
     if (start < end) httpdSend_orDie(conn, data + start, end - start);
@@ -502,12 +506,12 @@ int httpdSend_html(HttpdConnData *conn, const char *data, int len)
 }
 
 /* encode for JS. returns 0 or 1 - 1 = success */
-int httpdSend_js(HttpdConnData *conn, const char *data, int len)
+int httpdSend_js(HttpdConnData *conn, const uint8_t *data, ssize_t len)
 {
     int start = 0, end = 0;
-    char c;
+    uint8_t c;
     if (conn->conn == NULL) { return 0; }
-    if (len < 0) { len = (int) strlen(data); }
+    if (len < 0) { len = (int) strlen((const char *) data); }
     if (len == 0) { return 0; }
 
     for (end = 0; end < len; end++) {
@@ -522,13 +526,13 @@ int httpdSend_js(HttpdConnData *conn, const char *data, int len)
             start = end + 1;
         }
 
-        if (c == '"') httpdSend_orDie(conn, "\\\"", 2);
-        else if (c == '\'') httpdSend_orDie(conn, "\\'", 2);
-        else if (c == '\\') httpdSend_orDie(conn, "\\\\", 2);
-        else if (c == '<') httpdSend_orDie(conn, "\\u003C", 6);
-        else if (c == '>') httpdSend_orDie(conn, "\\u003E", 6);
-        else if (c == '\n') httpdSend_orDie(conn, "\\n", 2);
-        else if (c == '\r') httpdSend_orDie(conn, "\\r", 2);
+        if (c == '"') httpdSendStr_orDie(conn, "\\\"");
+        else if (c == '\'') httpdSendStr_orDie(conn, "\\'");
+        else if (c == '\\') httpdSendStr_orDie(conn, "\\\\");
+        else if (c == '<') httpdSendStr_orDie(conn, "\\u003C");
+        else if (c == '>') httpdSendStr_orDie(conn, "\\u003E");
+        else if (c == '\n') httpdSendStr_orDie(conn, "\\n");
+        else if (c == '\r') httpdSendStr_orDie(conn, "\\r");
     }
 
     if (start < end) httpdSend_orDie(conn, data + start, end - start);
@@ -546,7 +550,7 @@ bool httpdFlushSendBuffer(HttpdConnData *conn)
     if (conn->priv->chunkHdr != NULL) {
         //We're sending chunked data, and the chunk needs fixing up.
         //Finish chunk with cr/lf
-        httpdSend(conn, "\r\n", 2);
+        httpdSendStr(conn, "\r\n");
         //Calculate length of chunk
         len = ((&conn->priv->sendBuff[conn->priv->sendBuffLen]) - conn->priv->chunkHdr) - 8;
         //Fix up chunk header to correct value
@@ -708,7 +712,7 @@ static void httpdProcessRequest(HttpdConnData *conn)
         //Look up URL in the built-in URL table.
         while (builtInUrls[i].url != NULL) {
             int match = 0;
-            const char const *route = builtInUrls[i].url;
+            const char *route = builtInUrls[i].url;
             //See if there's a literal match
             if (streq(route, conn->url)) { match = 1; }
             //See if there's a wildcard match (*)

spritehttpd/src/httpdespfs.c

@@ -86,7 +86,7 @@ serveStaticFile(HttpdConnData *connData, const char *filepath)
 {
     EspFsFile *file = connData->cgiData;
     int len;
-    char buff[FILE_CHUNK_LEN + 1];
+    uint8_t buff[FILE_CHUNK_LEN + 1];
     char acceptEncodingBuffer[64 + 1];
     int isGzip;
 
@@ -127,7 +127,7 @@ serveStaticFile(HttpdConnData *connData, const char *filepath)
             httpdGetHeader(connData, "Accept-Encoding", acceptEncodingBuffer, 64);
             if (strstr(acceptEncodingBuffer, "gzip") == NULL) {
                 //No Accept-Encoding: gzip header present
-                httpdSend(connData, gzipNonSupportedMessage, -1);
+                httpdSendStr(connData, gzipNonSupportedMessage);
                 espFsClose(file);
                 return HTTPD_CGI_DONE;
             }
@@ -202,9 +202,13 @@ tplSend(HttpdConnData *conn, const char *str, int len)
     if (conn == NULL) { return 0; }
     TplData *tpd = conn->cgiData;
 
-    if (tpd == NULL || tpd->tokEncode == ENCODE_PLAIN) { return httpdSend(conn, str, len); }
-    if (tpd->tokEncode == ENCODE_HTML) { return httpdSend_html(conn, str, len); }
-    if (tpd->tokEncode == ENCODE_JS) { return httpdSend_js(conn, str, len); }
+    if (tpd == NULL || tpd->tokEncode == ENCODE_PLAIN) {
+        return httpdSendStrN(conn, str, len);
+    } else if (tpd->tokEncode == ENCODE_HTML) {
+        return httpdSend_html(conn, (const uint8_t *) str, len);
+    } else if (tpd->tokEncode == ENCODE_JS) {
+        return httpdSend_js(conn, (const uint8_t *) str, len);
+    }
     return 0;
 }
 
@@ -281,7 +285,7 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *connData)
         sp = tpd->buff_sp;
         x = tpd->buff_x;
     } else {
-        len = espFsRead(tpd->file, buff, FILE_CHUNK_LEN);
+        len = espFsRead(tpd->file, (uint8_t *) buff, FILE_CHUNK_LEN);
         tpd->buff_len = len;
 
         e = buff;
@@ -295,7 +299,7 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *connData)
                 //Inside ordinary text.
                 if (buff[x] == '%') {
                     //Send raw data up to now
-                    if (sp != 0) { httpdSend(connData, e, sp); }
+                    if (sp != 0) { httpdSendStrN(connData, e, sp); }
                     sp = 0;
                     //Go collect token chars.
                     tpd->tokenPos = 0;
@@ -307,7 +311,7 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *connData)
                     if (tpd->tokenPos == 0) {
                         //This is the second % of a %% escape string.
                         //Send a single % and resume with the normal program flow.
-                        httpdSend(connData, "%", 1);
+                        httpdSendStrN(connData, "%", 1);
                     } else {
                         if (!tpd->chunk_resume) {
                             //This is an actual token.
@@ -365,12 +369,12 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *connData)
                          c != '.' && c != '_' && c != '-' && c != ':'
                         )) {
                         // looks like we collected some garbage
-                        httpdSend(connData, "%", 1);
+                        httpdSendStrN(connData, "%", 1);
                         if (tpd->tokenPos > 0) {
-                            httpdSend(connData, tpd->token, tpd->tokenPos);
+                            httpdSendStrN(connData, tpd->token, tpd->tokenPos);
                         }
                         // the bad char
-                        httpdSend(connData, &c, 1);
+                        httpdSendStrN(connData, &c, 1);
 
                         //Go collect normal chars again.
                         e = &buff[x + 1];
@@ -389,7 +393,7 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *connData)
     }
 
     //Send remaining bit.
-    if (sp != 0) { httpdSend(connData, e, sp); }
+    if (sp != 0) { httpdSendStrN(connData, e, sp); }
     if (len != FILE_CHUNK_LEN) {
         //We're done.
         ((TplCallback) (connData->cgiArg))(connData, NULL, &tpd->tplArg);