super polished fs tool

master
Ondřej Hruška 2 years ago
parent b60d333f69
commit a0b603c703
  1. 3
      .gitignore
  2. 23
      Makefile
  3. 9913
      demo/staticfiles.embed.c
  4. 8
      fstool/httpd-logging.h
  5. 558
      fstool/main.c
  6. 68
      fstool/parsing.c
  7. 7
      fstool/testfiles/LIMECURD.TXT
  8. BIN
      fstool/testfiles/mouse.jpg
  9. 2
      fstool/testout/.gitignore
  10. 24
      spritehttpd/lib/espfs/espfs.c
  11. 42
      spritehttpd/lib/espfs/espfsformat.h
  12. 10
      spritehttpd/src/cgi-espfs.c

3
.gitignore vendored

@ -7,6 +7,5 @@
*~
*.a
spritehttpd-demo
staticfiles-embed.c
staticfiles.bin
demo/staticfiles.fs.c
espfstool

@ -107,9 +107,11 @@ $(LIB_TARGET): $(LIB_OBJS)
# -*- Demo -*-
# these won't have dir prefix added - they can refer to other dirs
DEMO_FS_C_FILE = $(DEMO_DIR)/staticfiles.fs.c
DEMO_SOURCES = \
$(DEMO_DIR)/server_demo.c \
$(DEMO_DIR)/staticfiles.bin.c
$(DEMO_FS_C_FILE)
DEMO_INCLUDE_DIRS = \
$(DEMO_DIR) \
@ -124,17 +126,11 @@ DEMO_OBJS = $(DEMO_SOURCES:.c=.o)
DEMO_STATIC_FILES := $(addprefix $(DEMO_DIR)/staticfiles/, $(DEMO_STATIC_FILES))
DEMO_CFLAGS += $(addprefix -I, $(DEMO_INCLUDE_DIRS))
demo/staticfiles.bin.c: $(FSTOOL_TARGET) $(DEMO_STATIC_FILES)
# Create the FS image
$(FSTOOL_TARGET) \
--compress=1 \
--gzip-exts=jpg \
--output=demo/staticfiles.bin \
--c-output=demo/staticfiles.bin.c \
--strip-path=demo/staticfiles \
$(DEMO_STATIC_FILES)
# Show content of the image
$(FSTOOL_TARGET) -p demo/staticfiles.bin
$(DEMO_FS_C_FILE): $(FSTOOL_TARGET) $(DEMO_STATIC_FILES)
# Create the FS image & convert it to a C file for embedding in the binary.
# This can be split to two commands if you want to inspect the binary image (specify output and input file instead of -)
$(FSTOOL_TARGET) -P --gzip=jpg -o - --strip=$(DEMO_DIR)/staticfiles $(DEMO_STATIC_FILES) \
| $(FSTOOL_TARGET) -M -o $@ -i -
$(DEMO_TARGET): $(DEMO_SOURCES) $(LIB_TARGET)
$(CC) $(DEMO_CFLAGS) $(DEMO_SOURCES) \
@ -164,10 +160,9 @@ $(FSTOOL_TARGET): $(FSTOOL_SOURCES)
# -*- Clean -*-
clean:
rm -f $(FSTOOL_TARGET) $(DEMO_TARGET) $(LIB_TARGET)
rm -f $(FSTOOL_TARGET) $(DEMO_TARGET) $(LIB_TARGET) $(DEMO_FS_C_FILE)
find . -type f \( \
-name '*.o' \
-o -name '*.embed.c' \
-o -name '*.d' \
-o -name '*.a' \
-o -name '*.elf' \

File diff suppressed because it is too large Load Diff

@ -3,7 +3,7 @@
#include <stdio.h>
#ifndef VERBOSE_LOGGING
#define VERBOSE_LOGGING 1
#define VERBOSE_LOGGING 0
#endif
#ifndef LOG_EOL
@ -51,7 +51,7 @@
*/
#define info(fmt, ...) \
do { \
fprintf(stderr, "[i] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
fprintf(stderr, "\x1b[37;1m[i] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
} while(0)
#else
#define dbg(fmt, ...)
@ -64,7 +64,7 @@
*/
#define error(fmt, ...) \
do { \
fprintf(stderr, "[E] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
fprintf(stderr, "\x1b[31;1m[E] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
} while(0)
/**
@ -73,7 +73,7 @@
*/
#define warn(fmt, ...) \
do { \
fprintf(stderr, "[W] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
fprintf(stderr, "\x1b[33;1m[W] " fmt "\x1b[0m"LOG_EOL, ##__VA_ARGS__); \
} while(0)
// --------------- logging categories --------------------

@ -8,14 +8,18 @@
#include <zlib.h>
#include <getopt.h>
#include <stdbool.h>
#include <errno.h>
#include "espfsformat.h"
#include "heatshrink_encoder.h"
#include "parsing.h"
#include "httpd-logging.h"
#define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif"
void show_version(char **argv);
void show_help(int retval, char **argv);
struct InputFileLinkedListEntry;
#define DEFAULT_GZIP_EXTS "css,js,svg,png,jpg,jpeg,webm,ico,gif"
#define DEFAULT_C_VARNAME "espfs_image"
struct InputFileLinkedListEntry {
char *name;
@ -27,7 +31,7 @@ static struct InputFileLinkedListEntry *s_inputFiles = NULL;
static struct InputFileLinkedListEntry *s_lastInputFile = NULL;
/// Output file FD
static int s_outFd = 1;
static int s_outFd = STDOUT_FILENO;
/// Array of gzipped extensions, ends with a NULL pointer
static char **s_gzipExtensions = NULL;
@ -69,7 +73,7 @@ size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t
level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4
heatshrink_encoder *enc = heatshrink_encoder_alloc(ws[level], ls[level]);
if (enc == NULL) {
perror("allocating mem for heatshrink");
espfs_error("allocating mem for heatshrink");
exit(1);
}
//Save encoder parms as first byte
@ -96,7 +100,7 @@ size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t
} while (insize != 0);
if (insize != 0) {
fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", (int) insize, sres, pres);
espfs_error("Heatshrink: Bug? insize is still %d. sres=%d pres=%d", (int) insize, sres, pres);
exit(1);
}
@ -111,10 +115,9 @@ size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t
* @param insize - len of the uncompressed input
* @param[out] out - destination buffer for the compressed data
* @param outcap - capacity of the output buffer
* @param level - compression level, 1-9; -1 for default.
* @return actual length of the compressed data
*/
size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap)
{
z_stream stream;
int zresult;
@ -127,21 +130,21 @@ size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outca
stream.next_out = out;
stream.avail_out = (uInt) outcap;
// 31 -> 15 window bits + 16 for gzip
zresult = deflateInit2(&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
zresult = deflateInit2(&stream, Z_BEST_COMPRESSION /* we want the smallest possible files */, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
if (zresult != Z_OK) {
fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult);
espfs_error("DeflateInit2 failed with code %d", zresult);
exit(1);
}
zresult = deflate(&stream, Z_FINISH);
if (zresult != Z_STREAM_END) {
fprintf(stderr, "Deflate failed with code %d\n", zresult);
espfs_error("Deflate failed with code %d", zresult);
exit(1);
}
zresult = deflateEnd(&stream);
if (zresult != Z_OK) {
fprintf(stderr, "DeflateEnd failed with code %d\n", zresult);
espfs_error("DeflateEnd failed with code %d", zresult);
exit(1);
}
@ -239,31 +242,31 @@ int handleFile(int fd, const char *name, int compression_mode, int level, const
csize = 100;
} // enlarge buffer if this is the case
cdat = cdatbuf = malloc(csize);
csize = compressGzip(fdat, size, cdat, csize, level);
compression_mode = COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger
flags = FLAG_GZIP;
} else if (compression_mode == COMPRESS_NONE) {
csize = compressGzip(fdat, size, cdat, csize);
compression_mode = EFS_COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger
flags = EFS_FLAG_GZIP;
} else if (compression_mode == EFS_COMPRESS_NONE) {
csize = size;
cdat = fdat;
} else if (compression_mode == COMPRESS_HEATSHRINK) {
} else if (compression_mode == EFS_COMPRESS_HEATSHRINK) {
cdat = cdatbuf = malloc(size * 2);
csize = compressHeatshrink(fdat, size, cdat, size * 2, level);
} else {
fprintf(stderr, "Unknown compression - %d\n", compression_mode);
espfs_error("Unknown compression - %d", compression_mode);
exit(1);
}
if (csize > size) {
fprintf(stderr, "! Compression enbiggened %s, embed as plain\n", name);
espfs_dbg("! Compression enbiggened %s, embed as plain", name);
//Compressing enbiggened this file. Revert to uncompressed store.
compression_mode = COMPRESS_NONE;
compression_mode = EFS_COMPRESS_NONE;
csize = size;
cdat = fdat;
flags = 0;
}
//Fill header data
h.magic = htole32(ESPFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
h.magic = htole32(EFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
h.flags = flags;
h.compression = (uint8_t) compression_mode;
h.nameLen = realNameLen = (uint16_t) strlen(name) + 1; // zero terminator
@ -299,10 +302,10 @@ int handleFile(int fd, const char *name, int compression_mode, int level, const
// debug outputs ...
if (compName != NULL) {
if (h.compression == COMPRESS_HEATSHRINK) {
if (h.compression == EFS_COMPRESS_HEATSHRINK) {
*compName = "heatshrink";
} else if (h.compression == COMPRESS_NONE) {
if (h.flags & FLAG_GZIP) {
} else if (h.compression == EFS_COMPRESS_NONE) {
if (h.flags & EFS_FLAG_GZIP) {
*compName = "gzip";
} else {
*compName = "none";
@ -322,9 +325,9 @@ int handleFile(int fd, const char *name, int compression_mode, int level, const
void finishArchive(void)
{
EspFsHeader h;
h.magic = htole32(ESPFS_MAGIC);
h.flags = FLAG_LASTFILE;
h.compression = COMPRESS_NONE;
h.magic = htole32(EFS_MAGIC);
h.flags = EFS_FLAG_LASTFILE;
h.compression = EFS_COMPRESS_NONE;
h.nameLen = 0;
h.fileLenComp = 0;
h.fileLenDecomp = 0;
@ -339,7 +342,7 @@ void finishArchive(void)
*/
void queueInputFile(const char *name)
{
fprintf(stderr, "INFILE: %s\n", name);
espfs_dbg("INFILE: %s", name);
struct InputFileLinkedListEntry *tmp = malloc(sizeof(struct InputFileLinkedListEntry));
tmp->name = strdup(name);
@ -354,48 +357,70 @@ void queueInputFile(const char *name)
}
}
int main(int argc, char **argv)
enum OpMode {
OM_INVALID = 0,
OM_PACK = 'P',
OM_LIST = 'L',
OM_EXTRACT = 'X',
OM_EMBED = 'M',
};
#define BUFLEN 1024
int main(const int argc, char **argv)
{
int f;
char inputFileName[1024];
char inputFileName[BUFLEN];
char tempbuf[BUFLEN];
struct stat statBuf;
int serr;
int rate;
int err = 0;
int compType; //default compression type - heatshrink
int compLvl = -1;
bool use_gzip = false;
enum OpMode opmode = OM_INVALID;
compType = COMPRESS_HEATSHRINK;
compType = EFS_COMPRESS_HEATSHRINK;
int c;
char *outfile = NULL;
char *c_outfile = NULL;
char *outFileName = NULL;
char *c_varname = NULL;
char *parseFile = NULL;
char *stripPath = NULL;
char *extractFile = NULL;
char *extractFileName = NULL;
size_t num_input_files = 0;
bool read_from_stdin = false;
while (1) {
int option_index = 0;
static struct option long_options[] = {
{"parse", required_argument, 0, 'p'},
{"extract", required_argument, 0, 'e'},
{"compress", required_argument, 0, 'c'},
{"gzip", no_argument, 0, 'z'},
{"gzip-all", no_argument, 0, 'G'},
{"level", required_argument, 0, 'l'},
{"gzip-exts", required_argument, 0, 'g'},
{"input", required_argument, 0, 'i'},
{"output", required_argument, 0, 'o'},
{"c-output", required_argument, 0, 'C'},
{"c-varname", required_argument, 0, 'N'},
{"strip-path", required_argument, 0, 'S'},
// Help
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'V'},
// Main operation (one at a time)
{"pack", no_argument, 0, 'P'},
{"list", no_argument, 0, 'L'},
{"extract", required_argument, 0, 'X'},
{"embed", no_argument, 0, 'M'},
// Common options
{"input", required_argument, 0, 'i'}, // input file; "-" to read them as lines on stdin; can be repeated in case of --pack.
// Input files can also be given as stand-alone arguments at the end, e.g. as a result of glob
{"output", required_argument, 0, 'o'}, // output file; "-" for stdout (default)
// Options for --pack
{"compress", required_argument, 0, 'c'}, // 0 = no, 1 = heatshrink (def)
{"level", required_argument, 0, 'l'}, // Heatshrink compression level 1-9, -1 = default
{"gzip", optional_argument, 0, 'z'}, // Gzipped extensions; no arg = default, "*" = all, comma-separated list = custom exts
{"strip", required_argument, 0, 's'}, // path removed from all input files, e.g. when they are in a subfolder
// Options for --embed
{"varname", required_argument, 0, 'n'}, // name of the array; the length variable is {varname}_len
{ /* end marker */ }
};
c = getopt_long(argc, argv, "c:l:g:zGS:e:hp:C:i:o:0123456789",
c = getopt_long(argc, argv, "h?VPLX:Mi:o:c:l:z::s:n:",
long_options, &option_index);
if (c == -1) {
break;
@ -403,223 +428,356 @@ int main(int argc, char **argv)
switch (c) {
case 'h':
goto show_help;
case '?':
show_help(0, argv);
case 'z':
use_gzip = true;
break;
case 'V':
show_version(argv);
case '0' ... '9':
compLvl = c - '0';
case 'P':
opmode = OM_PACK;
break;
case 'p':
parseFile = strdup(optarg);
case 'L':
opmode = OM_LIST;
break;
case 'e':
extractFile = strdup(optarg);
case 'X':
opmode = OM_EXTRACT;
if (extractFileName) {
espfs_error("can extract only one file at a time!");
exit(1);
}
extractFileName = strdup(optarg);
break;
case 'C':
c_outfile = strdup(optarg);
case 'M':
opmode = OM_EMBED;
break;
case 'N':
c_varname = strdup(optarg);
case 'i':
if (0 == strcmp(optarg, "-")) {
read_from_stdin = true;
} else {
queueInputFile(optarg);
}
num_input_files++;
break;
case 'S':
stripPath = strdup(optarg);
case 'o':
outFileName = strdup(optarg);
break;
case 'c':
compType = atoi(optarg);
errno = 0;
compType = (int) strtol(optarg, NULL, 10);
if (errno != 0 || compType < 0 || compType > 1) {
espfs_error("Bad compression mode: %s", optarg);
exit(1);
}
break;
case 'G':
use_gzip = true;
s_gzipAll = true;
case 'l':
errno = 0;
compLvl = (int) strtol(optarg, NULL, 10);
if (errno != 0 || compLvl < 1 || compLvl > 9) {
espfs_error("Bad compression level: %s", optarg);
exit(1);
}
break;
case 'g':
case 'z':
use_gzip = true;
parseGzipExtensions(optarg);
break;
case 'l':
compLvl = atoi(optarg);
if (compLvl < 1 || compLvl > 9) {
fprintf(stderr, "Bad compression level: %d\n", compLvl);
err = 1;
goto show_help;
if (optarg) {
if (0 == strcmp("*", optarg)) {
s_gzipAll = true;
} else {
parseGzipExtensions(optarg);
}
} else {
parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS)); // memory leak! ehh
}
break;
case 'i':
queueInputFile(optarg);
case 's':
stripPath = strdup(optarg);
break;
case 'o':
outfile = strdup(optarg);
case 'n':
c_varname = strdup(optarg);
break;
case '?':
goto show_help;
default:
fprintf(stderr, "Unknown option: %c\n", c);
err = 1;
goto show_help;
espfs_error("Unknown option: %c", c);
exit(1);
}
}
FILE *outfp = NULL;
if (outfile && 0 != strcmp("-", outfile)) {
fprintf(stderr, "Writing to %s\n", outfile);
outfp = fopen(outfile, "w+");
if (!outfp) {
perror(outfile);
return 1;
}
s_outFd = fileno(outfp);
ftruncate(s_outFd, 0);
} else {
if (outfile) {
free(outfile);
outfile = NULL;
}
fprintf(stderr, "Writing to stdout\n\n");
if (!use_gzip) {
s_gzipExtensions = NULL;
}
if (parseFile) {
parseEspfsImage(parseFile, extractFile, s_outFd);
exit(0);
}
bool want_output;
bool allows_multiple_inputs;
switch (opmode) {
case OM_PACK:
want_output = true;
allows_multiple_inputs = true;
break;
if (s_gzipExtensions == NULL && use_gzip) {
parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS));
case OM_LIST:
want_output = false;
allows_multiple_inputs = false;
break;
case OM_EXTRACT:
case OM_EMBED:
want_output = true;
allows_multiple_inputs = false;
break;
default:
espfs_error("Specify one of the operation modes: -P, -L, -X, -M");
exit(1);
}
if (optind < argc) {
while (optind < argc) {
queueInputFile(argv[optind++]);
/* Input */
while (optind < argc) {
char *s = argv[optind++];
if (0 == strcmp(s, "-")) {
read_from_stdin = true;
} else {
queueInputFile(s);
num_input_files++;
}
}
if (!s_inputFiles) {
fprintf(stderr, "Reading input file names from stdin\n");
if (num_input_files == 0 && read_from_stdin && opmode == OM_PACK) {
read_from_stdin = false;
espfs_dbg("Reading input file names from stdin");
while (fgets(inputFileName, sizeof(inputFileName), stdin)) {
//Kill off '\n' at the end
inputFileName[strlen(inputFileName) - 1] = 0;
queueInputFile(inputFileName);
num_input_files++;
}
}
struct InputFileLinkedListEntry *entry = s_inputFiles;
while (entry) {
char *name = entry->name;
//Only include files
serr = stat(name, &statBuf);
if ((serr == 0) && S_ISREG(statBuf.st_mode)) {
//Strip off './' or '/' madness.
char *embeddedName = name;
f = open(name, O_RDONLY);
// relative path starting with ./, remove that
if (embeddedName[0] == '.' && embeddedName[1] == '/') {
embeddedName += 2;
}
// remove prefix
if (stripPath && 0 == strncmp(embeddedName, stripPath, strlen(stripPath))) {
embeddedName += strlen(stripPath);
}
// remove leading slash, if any
if (embeddedName[0] == '/') { embeddedName++; }
if (f > 0) {
const char *compName = "unknown";
rate = handleFile(f, embeddedName, compType, compLvl, &compName);
fprintf(stderr, "%s (%d%%, %s)\n", embeddedName, rate, compName);
close(f);
if (!read_from_stdin) {
if (num_input_files == 0) {
if (allows_multiple_inputs) {
espfs_error("Specify input file(s)!");
} else {
perror(name);
espfs_error("Specify input file!");
}
} else if (serr != 0) {
perror(name);
exit(1);
} else if (!allows_multiple_inputs && num_input_files > 1) {
espfs_error("Mode %c requires exactly one input file!", opmode);
exit(1);
}
}
char *inFileName = read_from_stdin ? NULL : s_inputFiles->name;
entry = entry->next;
/* Output */
if (!want_output && outFileName) {
espfs_error("Output file is not allowed in %c mode!", opmode);
exit(1);
}
finishArchive();
fsync(s_outFd);
FILE *outFile = NULL;
bool write_to_stdout = outFileName && (0 == strcmp("-", outFileName));
if (outFileName && !write_to_stdout) {
espfs_dbg("Writing to %s", outFileName);
outFile = fopen(outFileName, "w+");
if (!outFile) {
perror(outFileName);
return 1;
}
s_outFd = fileno(outFile);
ftruncate(s_outFd, 0);
} else {
if (outFileName) {
free(outFileName);
outFileName = NULL;
}
if (want_output && !write_to_stdout) {
espfs_error("Specify output file! Use -o - for stdout");
exit(1);
}
}
if (outfp) {
if (outfile && c_outfile) {
const size_t imagelen = (size_t) lseek(s_outFd, 0, SEEK_END);
lseek(s_outFd, 0, SEEK_SET);
FILE *coutf = fopen(c_outfile, "w+");
if (!coutf) {
perror(c_outfile);
return 1;
/* Do it! */
switch (opmode) {
case OM_PACK: {
struct InputFileLinkedListEntry *entry = s_inputFiles;
while (entry) {
char *name = entry->name;
//Only include files
serr = stat(name, &statBuf);
if ((serr == 0) && S_ISREG(statBuf.st_mode)) {
//Strip off './' or '/' madness.
char *embeddedName = name;
f = open(name, O_RDONLY);
if (f > 0) {
// relative path starting with ./, remove that
if (embeddedName[0] == '.' && embeddedName[1] == '/') {
embeddedName += 2;
}
// remove prefix
if (stripPath && 0 == strncmp(embeddedName, stripPath, strlen(stripPath))) {
embeddedName += strlen(stripPath);
}
// remove leading slash, if any
if (embeddedName[0] == '/') { embeddedName++; }
const char *compName = "unknown";
rate = handleFile(f, embeddedName, compType, compLvl, &compName);
(void) rate;
espfs_dbg("%s (%d%%, %s)", embeddedName, rate, compName);
close(f);
} else {
snprintf(tempbuf, BUFLEN, "Open %s for reading: %s", name, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
} else if (serr != 0) {
snprintf(tempbuf, BUFLEN, "Stat %s: %s", name, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
entry = entry->next;
}
finishArchive();
fsync(s_outFd);
if (outFile) {
fclose(outFile);
}
} break;
case OM_LIST: {
if (!inFileName) {
espfs_error("Input file required!");
exit(1);
}
parseEspfsImage(inFileName, NULL, s_outFd);
} break;
case OM_EXTRACT: {
if (!inFileName) {
espfs_error("Input file required!");
exit(1);
}
parseEspfsImage(inFileName, extractFileName, s_outFd);
} break;
case OM_EMBED: {
FILE *inFile = NULL;
int inFD = STDIN_FILENO;
if (!read_from_stdin) {
inFile = fopen(inFileName, "r");
if (!inFile) {
snprintf(tempbuf, BUFLEN, "Open %s for reading: %s", inFileName, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
inFD = fileno(inFile);
}
int cfd = fileno(coutf);
ftruncate(cfd, 0);
if (!c_varname) {
c_varname = strdup("espfs_image");
c_varname = strdup(DEFAULT_C_VARNAME);
}
fprintf(coutf, "unsigned char %s[%lu] = {", c_varname, imagelen);
for (size_t i = 0; i < imagelen; i++) {
fprintf(coutf, (i % 16) ? " " : "\n ");
uint8_t u;
read(s_outFd, &u, 1);
fprintf(coutf, "0x%02x,", u);
size_t len;
len = (size_t) snprintf(tempbuf, BUFLEN, "unsigned char %s[] = {", c_varname);
write(s_outFd, tempbuf, len);
uint8_t u;
size_t imageLen = 0;
while (1 == read(inFD, &u, 1)) {
len = (size_t) snprintf(tempbuf, BUFLEN, "%s0x%02x,", ((imageLen % 16) ? " " : "\n "), u);
write(s_outFd, tempbuf, len);
imageLen++;
}
fprintf(coutf, "\n};\nunsigned int %s_len = %lu;\n", c_varname, imagelen);
fsync(cfd);
fclose(coutf);
}
len = (size_t) snprintf(tempbuf, BUFLEN, "\n};\nunsigned int %s_len = %lu;\n", c_varname, imageLen);
write(s_outFd, tempbuf, len);
fclose(outfp);
fsync(s_outFd);
if (outFile) {
fclose(outFile);
}
if (inFile) {
fclose(inFile);
}
} break;
default:
__builtin_unreachable();
}
return 0;
}
__attribute__((noreturn))
void show_version(char **argv)
{
// to stdout
printf("%s #%s\n", argv[0], GIT_HASH);
exit(0);
}
__attribute__((noreturn))
void show_help(int retval, char **argv)
{
// to stderr, this can happen as a response to bad args
show_help:
// ##########**********##########**********##########----------$$$$$$$$$$----------$$$$$$$$$$----------| 80 chars
fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
fprintf(stderr, "Options:\n");
fprintf(stderr, "[-p|--parse FILE]\n"
" Parse an espfs file and show a list of its contents. No other options apply in this mode.\n");
fprintf(stderr, "[-e|--extract FILE]\n"
" Extract a file with the given name from the parsed file (-p)\n");
fprintf(stderr, "[-S|--strip-path PATH]\n"
" Remove the given path from input file names before packing\n");
fprintf(stderr, "[-c|--compress COMPRESSOR]\n"
" 0 - None, 1 - Heatshrink (default)\n");
fprintf(stderr, "[-l|--level LEVEL] or [-0 through -9]\n"
" compression level 1-9, higher is better but uses more RAM\n");
fprintf(stderr, "[-z|--gzip]\n"
" use gzip for files with extensions matching "DEFAULT_GZIP_EXTS"\n");
fprintf(stderr, "[-G|--gzip-all]\n"
" use gzip for all files\n");
fprintf(stderr, "[-g|--gzip-exts GZIPPED_EXTENSIONS]\n"
" use gzip for files with custom extensions, comma-separated\n");
fprintf(stderr, "[-i|--input FILE]\n"
" Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin\n"
" if not specified by args\n");
fprintf(stderr, "[-o|--output FILE]\n"
" Output file name; if not specified or equal to \"-\", outputs to stdout\n");
fprintf(stderr, "[-C|--c-output FILE]\n"
" C embed output file name (an uint8_t array for embedding); can be used only if a binary output\n"
" file is specified as well (using -o), as this reads the output file and converts it as a second step.\n");
fprintf(stderr, "[-h|--help\n"
" Show help.\n\n");
exit(err);
fprintf(stderr, "%s #%s - Program to create and parse espfs images\n", argv[0], GIT_HASH);
fprintf(stderr, "\n"
"One main operation mode must be selected:\n"
" -P, --pack Create a binary fs image\n"
" -L, --list Read a binary fs image and show the contents\n"
" -X, --extract=NAME Read a binary fs image and extract a file with the given name\n"
" -M, --embed Read a binary file (typically the binary fs image produced by -P) and\n"
" convert it to C syntax byte array and a constant with its length.\n"
" -h, -?, --help Show this help (has precedence over other options)\n"
"\n"
"Additional arguments specify the input, output and parameters:\n"
" -i, --input=FILE Input file name. Can be used multiple times. For convenience with globs,\n"
" input files can be given at the end of the option string without -i.\n"
" The \"-\" filename means \"read from stdin\":\n"
" - pack: reads file names to pack from stdin (e.g. piped from `find`)\n"
" - embed: read the binary file from stdin (e.g. piped from the --pack\n"
" option, avoiding the creation of a temporary binary file)\n"
" Stdin reading is not supported for options --extract and --list.\n"
"\n"
" -o, --output=FILE Output file, \"-\" for stdout.\n"
" Output can't be changed in --list mode.\n"
"\n"
"Pack options:\n"
" -c, --compress=MODE Compression mode, 0=none, 1=heatshrink (default). Not used if the file\n"
" is gzipped - additional compression wouldn't be useful.\n"
" -l, --level=LEVEL Compression level, 1-9, -1=default. 1=worst, 9=best compression, but uses\n"
" more RAM to unpack.\n"
" -s, --strip=PATH Strip a path prefix from all packed files (e.g. a subfolder name)\n"
" -z, --gzip[=EXTS] Enable gzip compression for some file types (filename extensions).\n"
" By default, enabled for "DEFAULT_GZIP_EXTS".\n"
" Set EXTS to * to enable gzip for all files. Custom extensions are set as\n"
" a comma-separated list.\n"
"\n"
"Embed options:\n"
" -n, --varname=VARNAME Set custom array name to use in the generated C code. The length constant\n"
" is called {VARNAME}_len. The default VARNAME is \""DEFAULT_C_VARNAME"\"\n"
);
exit(retval);
}

@ -3,9 +3,12 @@
#include <stdlib.h>
#include <unistd.h>
#include <zlib.h>
#include <errno.h>
#include <string.h>
#include "parsing.h"
#include "espfs.h"
#include "httpd-logging.h"
static off_t espfs_parse_filesize = -1;
static int espfs_parse_fd = -1;
@ -14,6 +17,8 @@ size_t decompressGzip(const uint8_t *in, size_t insize, int outfd)
{
#define OUTBUF_LEN 10240
uint8_t outbuf[OUTBUF_LEN];
#define ERRORBUF_LEN 256
char errorbuf[ERRORBUF_LEN];
z_stream stream;
int zresult;
@ -28,7 +33,7 @@ size_t decompressGzip(const uint8_t *in, size_t insize, int outfd)
// 31 -> 15 window bits + 16 for gzip
zresult = inflateInit2(&stream, 31);
if (zresult != Z_OK) {
fprintf(stderr, "InflateInit2 failed with code %d\n", zresult);
espfs_error("InflateInit2 failed with code %d", zresult);
exit(1);
}
@ -36,33 +41,34 @@ size_t decompressGzip(const uint8_t *in, size_t insize, int outfd)
stream.avail_out = OUTBUF_LEN;
stream.next_out = outbuf;
fprintf(stderr, "inflate chunk\n");
espfs_dbg("inflate chunk\n");
zresult = inflate(&stream, Z_FINISH);
if (zresult == Z_BUF_ERROR || zresult == Z_OK || zresult == Z_STREAM_END) {
size_t have = OUTBUF_LEN - stream.avail_out;
fprintf(stderr, "inflated: %d\n", (int) have);
espfs_dbg("inflated: %d\n", (int) have);
errno = 0;
if ((ssize_t)have != write(outfd, outbuf, have)) {
perror("Write output");
snprintf(errorbuf, ERRORBUF_LEN, "Write output: %s", strerror(errno));
espfs_error("%s", errorbuf);
exit(1);
}
if (zresult == Z_STREAM_END) {
fprintf(stderr, "Z_STREAM_END\n");
espfs_dbg("Z_STREAM_END\n");
break;
}
} else {
fprintf(stderr, "gzip error: %d\n", zresult);
espfs_error("gzip error: %d\n", zresult);
exit(1);
}
}
zresult = inflateEnd(&stream);
if (zresult != Z_OK) {
fprintf(stderr, "InflateEnd failed with code %d\n", zresult);
espfs_error("InflateEnd failed with code %d", zresult);
exit(1);
}
fprintf(stderr, "Total decoded = %d\n", (int) stream.total_out);
espfs_dbg("Total decoded = %d\n", (int) stream.total_out);
return stream.total_out;
}
@ -77,11 +83,15 @@ size_t decompressGzip(const uint8_t *in, size_t insize, int outfd)
void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
{
int rv;
fprintf(stderr, "Parsing: %s\n", imagefile);
espfs_dbg("Parsing: %s", imagefile);
#define ERRORBUF_LEN 256
char errorbuf[ERRORBUF_LEN];
errno = 0;
FILE *f = fopen(imagefile, "r");
if (!f) {
perror(imagefile);
snprintf(errorbuf, ERRORBUF_LEN, "Open %s for writing: %s", imagefile, strerror(errno));
espfs_error("%s", errorbuf);
exit(1);
}
int fd = fileno(f);
@ -91,9 +101,9 @@ void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
espfs_parse_fd = fd;
rv = espFsInit();
rv = (int) espFsInit();
if (rv != 0) {
fprintf(stderr, "Fail to init FS\n");
espfs_error("Fail to init espfs: %d", rv);
exit(1);
}
@ -102,29 +112,29 @@ void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
EspFsHeader hdr;
if (!efile) {
fprintf(stderr, "Fail to open file %s from image\n", extractfile);
espfs_error("Fail to open file %s from image", extractfile);
exit(1);
}
rv = espFsFileReadHeader(efile, &hdr);
if (rv != 0) {
fprintf(stderr, "Fail to read file header\n");
espfs_error("Fail to read file header");
exit(1);
}
bool isGzip = hdr.flags & FLAG_GZIP;
bool isGzip = hdr.flags & EFS_FLAG_GZIP;
size_t expected_readlen = isGzip ? hdr.fileLenComp : hdr.fileLenDecomp;
uint8_t *buff = malloc(expected_readlen);
size_t lenRead = espFsRead(efile, buff, expected_readlen);
if (lenRead != expected_readlen) {
fprintf(stderr, "Fail to read raw file from espfs image - read len %d", (int) lenRead);
espfs_error("Fail to read raw file from espfs image - read len %d", (int) lenRead);
exit(1);
}
if (isGzip) {
fprintf(stderr, "[EspFS] File is gzipped!");
espfs_dbg("File is gzipped!");
decompressGzip(buff, lenRead, outfd);
} else {
write(outfd, buff, lenRead);
@ -144,8 +154,22 @@ void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
char namebuf[1024];
while (espFsWalkNext(&walk, &header, namebuf, 1024, &offset)) {
fprintf(stderr, "at %04x: \"%s\", flags: %02x, comp: %s, compLen: %d, plainLen: %d\n", offset, namebuf, header.flags,
header.compression == 1 ? "HS" : "None", header.fileLenComp, header.fileLenDecomp);
// to stdout
printf("File @ 0x%06x: \"%s\", flags=0x%02x, comp=%d, ", offset, namebuf, header.flags, header.compression);
if (header.compression == 1) {
printf("heathshrink (cold %d -> unpacked %d bytes)\n",
header.fileLenComp, header.fileLenDecomp);
} else if (header.flags & EFS_FLAG_GZIP) {
printf("gzip (cold %d -> unpacked %d bytes)\n",
header.fileLenComp, header.fileLenDecomp);
} else {
if (header.fileLenComp != header.fileLenDecomp) {
espfs_error("Uncompressed, but fileLenComp != fileLenDecomp (%d, %d)", header.fileLenComp, header.fileLenDecomp);
}
printf("plain (%d bytes)\n", header.fileLenDecomp);
}
}
fclose(f);
@ -154,9 +178,9 @@ void parseEspfsImage(const char *imagefile, const char *extractfile, int outfd)
int httpdPlatEspfsRead(void *dest, size_t offset, size_t len)
{
// fprintf(stderr, "FS read @ %d, len %d\n", offset, (int) len);
espfs_dbg("FS read @ %d, len %d\n", offset, (int) len);
if ((off_t) (offset + len) > espfs_parse_filesize) {
fprintf(stderr, "Read out fo range!\n");
espfs_error("Read out fo range!");
return -1;
}
lseek(espfs_parse_fd, (off_t) offset, SEEK_SET);

@ -1,7 +0,0 @@
Fruit curd is a dessert spread and topping usually made with citrus fruit, such as lemon, lime, orange, or tangerine. Other flavor variations include passion fruit, mango, and berries such as raspberries, cranberries or blackberries. The basic ingredients are beaten egg yolks, sugar, fruit juice, and zest, which are gently cooked together until thick and then allowed to cool, forming a soft, smooth, intensely flavoured spread. Some recipes also include egg whites and/or butter.
In late 19th and early 20th century England, home-made lemon curd was traditionally served with bread or scones at afternoon tea as an alternative to jam, and as a filling for cakes, small pastries, and tarts. Homemade lemon curd was usually made in relatively small amounts as it did not keep as well as jam. In more modern times, larger quantities became possible because of the use of refrigeration. Commercially manufactured curds often contain additional preservatives and thickening agents.
Contemporary commercially made curds remain a popular spread for bread, scones, toast, waffles, crumpets, pancakes, cheesecake, or muffins. They can also be used as a flavoring for desserts or yoghurt. Lemon-meringue pie, made with lemon curd and topped with meringue, has been a popular dessert in Britain and the United States since the nineteenth century. Lemon curd can also have whipped cream folded into it for such uses as filling cream puffs.
Curds differ from pie fillings or custards in that they contain a higher proportion of juice and zest, which gives them a more intense flavor. Also, curds containing butter have a smoother and creamier texture than both pie fillings and custards, which contain little or no butter and use cornstarch or flour for thickening. Additionally, unlike custards, curds are not usually eaten on their own.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

@ -1,2 +0,0 @@
*
!.gitignore

@ -46,7 +46,7 @@ EspFsInitResult espFsInit()
EspFsHeader testHeader;
int rv;
rv = httpdPlatEspfsRead(&testHeader, 0, sizeof(EspFsHeader));
if (rv != 0 || testHeader.magic != ESPFS_MAGIC) {
if (rv != 0 || testHeader.magic != EFS_MAGIC) {
espfs_error("[EspFS] Invalid magic on first file header");
return ESPFS_INIT_RESULT_NO_IMAGE;
}
@ -89,12 +89,12 @@ bool espFsWalkNext(EspFsWalk *walk, EspFsHeader *header, char *namebuf, size_t n
return false;
}
if (header->magic != ESPFS_MAGIC) {
if (header->magic != EFS_MAGIC) {
espfs_error("[EspFS] Magic mismatch. EspFS image broken.");
return false;
}
if (header->flags & FLAG_LASTFILE) {
if (header->flags & EFS_FLAG_LASTFILE) {
espfs_dbg("[EspFS] End of image.");
return false;
}
@ -126,11 +126,11 @@ bool espFsWalkNext(EspFsWalk *walk, EspFsHeader *header, char *namebuf, size_t n
EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos)
{
int rv;
if (h->magic != ESPFS_MAGIC) {
if (h->magic != EFS_MAGIC) {
espfs_error("[EspFS] Magic mismatch. EspFS image broken.");
return NULL;
}
if (h->flags & FLAG_LASTFILE) {
if (h->flags & EFS_FLAG_LASTFILE) {
espfs_dbg("[EspFS] End of image.");
return NULL;
}
@ -148,10 +148,10 @@ EspFsFile *espFsOpenFromHeader(EspFsHeader *h, uint32_t hpos)
espfs_dbg("[EspFS] Found file @ hpos %d", hpos);
if (h->compression == COMPRESS_NONE) {
if (h->compression == EFS_COMPRESS_NONE) {
r->decompData = NULL;
return r;
} else if (h->compression == COMPRESS_HEATSHRINK) {
} else if (h->compression == EFS_COMPRESS_HEATSHRINK) {
//File is compressed with Heatshrink.
char parm;
//Decoder params are stored in 1st byte.
@ -216,11 +216,11 @@ EspFsFile *espFsOpen(const char *fileName)
h.fileLenComp = le32toh(h.fileLenComp);
h.fileLenDecomp = le32toh(h.fileLenDecomp);
if (h.magic != ESPFS_MAGIC) {
if (h.magic != EFS_MAGIC) {
espfs_error("[EspFS] Magic mismatch. EspFS image broken.");
return NULL;
}
if (h.flags & FLAG_LASTFILE) {
if (h.flags & EFS_FLAG_LASTFILE) {
espfs_dbg("[EspFS] End of image.");
return NULL;
}
@ -264,7 +264,7 @@ size_t espFsRead(EspFsFile *fh, uint8_t *buff, size_t buf_cap)
//Cache file length.
//Do stuff depending on the way the file is compressed.
if (fh->decompressor == COMPRESS_NONE) {
if (fh->decompressor == EFS_COMPRESS_NONE) {
int toRead = (int) binary_len - (int) (fh->posComp - fh->posStart);
if (toRead < 0) { toRead = 0; }
if ((int) buf_cap > toRead) { buf_cap = (size_t) toRead; }
@ -277,7 +277,7 @@ size_t espFsRead(EspFsFile *fh, uint8_t *buff, size_t buf_cap)
fh->posDecomp += buf_cap;
fh->posComp += buf_cap;
return buf_cap;
} else if (fh->decompressor == COMPRESS_HEATSHRINK) {
} else if (fh->decompressor == EFS_COMPRESS_HEATSHRINK) {
rv = httpdPlatEspfsRead(&decompressed_len, fh->headerPos + offsetof(EspFsHeader, fileLenDecomp), 4);
if (rv != 0) {
return 0;
@ -335,7 +335,7 @@ size_t espFsRead(EspFsFile *fh, uint8_t *buff, size_t buf_cap)
void espFsClose(EspFsFile *fh)
{
if (fh == NULL) { return; }
if (fh->decompressor == COMPRESS_HEATSHRINK) {
if (fh->decompressor == EFS_COMPRESS_HEATSHRINK) {
heatshrink_decoder *dec = (heatshrink_decoder *) fh->decompData;
heatshrink_decoder_free(dec);
}

@ -1,5 +1,12 @@
#pragma once
/*
Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module.
Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files,
headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself
when trying to do a <4byte or unaligned read.
*/
/*
The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data.
Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header
@ -8,18 +15,31 @@ with the FLAG_LASTFILE flag set.
#include <stdint.h>
#define FLAG_LASTFILE (1<<0)
#define FLAG_GZIP (1<<1)
#define COMPRESS_NONE 0
#define COMPRESS_HEATSHRINK 1
#define ESPFS_MAGIC 0x73665345 /* ASCII sequence of bytes 'E' 'S' 'f' 's' interpreted as a little endian uint32_t */
/// Last file in the filesystem image
#define EFS_FLAG_LASTFILE (1<<0)
/// Gzipped file. This does not affect espfs, it's a user-level flag; un-gzip is not implemented as part
/// of the FS or CGI - gzipped files are sent directly to the browser with the appropriate header -
/// saving bandwidth and getting us better compression ratios than heatshrink can achieve for larger files.
#define EFS_FLAG_GZIP (1<<1)
/// No filesystem level compression
#define EFS_COMPRESS_NONE 0
/// The file is stored as compressed
#define EFS_COMPRESS_HEATSHRINK 1
/// Magic number leading each file record
#define EFS_MAGIC 0x73665345 /* ASCII sequence of bytes 'E' 'S' 'f' 's' interpreted as a little endian uint32_t */
/* 16 bytes long for alignment */
typedef struct {
uint32_t magic;
uint8_t flags;
uint8_t compression;
uint16_t nameLen;
uint32_t fileLenComp;
uint32_t fileLenDecomp;
/// Magic number
uint32_t magic;
/// File flags
uint8_t flags;
/// Filesystem-level compression used
uint8_t compression;
/// Length of the file name
uint16_t nameLen;
/// Compressed file length (differs from the decompressed size if either gzip or heatshrink is used)
uint32_t fileLenComp;
/// Decompressed file length
uint32_t fileLenDecomp;
} __attribute__((packed)) EspFsHeader;

@ -21,7 +21,7 @@ Connector to let httpd use the espfs filesystem to serve the files in it.
#include "espfs.h"
#include "espfsformat.h"
#define FILE_CHUNK_LEN 1024
#define FILE_CHUNK_LEN 512
// The static files marked with FLAG_GZIP are compressed and will be served with GZIP compression.
// If the client does not advertise that he accepts GZIP send following warning message (telnet users for e.g.)
@ -88,7 +88,6 @@ static httpd_cgi_state serveStaticFile(HttpdConnData *hconn, const char *filepat
size_t len;
uint8_t buff[FILE_CHUNK_LEN + 1];
char acceptEncodingBuffer[64 + 1];
int isGzip;
if (hconn->conn == NULL) {
//Connection aborted. Clean up.
@ -120,7 +119,7 @@ static httpd_cgi_state serveStaticFile(HttpdConnData *hconn, const char *filepat
// If there are no gzipped files in the image, the code bellow will not cause any harm.
// Check if requested file was GZIP compressed
isGzip = espFsFlags(file) & FLAG_GZIP;
bool isGzip = (0 != (espFsFlags(file) & EFS_FLAG_GZIP));
if (isGzip) {
// Check the browser's "Accept-Encoding" header. If the client does not
// advertise that he accepts GZIP send a warning message (telnet users for e.g.)
@ -264,7 +263,7 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *conn)
}
}
if (espFsFlags(tdi->file) & FLAG_GZIP) {
if (espFsFlags(tdi->file) & EFS_FLAG_GZIP) {
espfs_error("cgiEspFsTemplate: Trying to use gzip-compressed file %s as template!", conn->url);
espFsClose(tdi->file);
httpdPlatFree(tdi);
@ -274,11 +273,14 @@ httpd_cgi_state cgiEspFsTemplate(HttpdConnData *conn)
tdi->tokenPos = -1;
conn->cgiData = tdi;
httpdStartResponse(conn, 200);
// Try to extension from the URL
const char *mime = httpdGetMimetype(conn->url);
if (!mime) {
// Get extension from the template file
mime = httpdGetMimetype(filepath);
}
if (!mime) {
// Still unresolved...
mime = "text/html"; // this is generally a good fallback for templates
}

Loading…
Cancel
Save