SpriteHTTPD - embedded HTTP server with read-only filesystem and templating, originally developed for ESP8266, now stand-alone and POSIX compatible.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
spritehttpd/espfsbuilder/main.c

559 lines
16 KiB

#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zlib.h>
#include <getopt.h>
#include <stdbool.h>
#include "espfsformat.h"
#include "heatshrink_encoder.h"
#include "parsing.h"
#define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif"
struct InputFileLinkedListEntry;
struct InputFileLinkedListEntry {
char *name;
struct InputFileLinkedListEntry *next;
};
/// two ends of a linked list with input files
static struct InputFileLinkedListEntry *s_inputFiles = NULL;
static struct InputFileLinkedListEntry *s_lastInputFile = NULL;
/// Output file FD
static int s_outFd = 1;
/// Array of gzipped extensions, ends with a NULL pointer
static char **s_gzipExtensions = NULL;
/// Gzip all files
static bool s_gzipAll = false;
// impls to satisfy defs in the config header
void *httpdPlatMalloc(size_t len)
{
return malloc(len);
}
void httpdPlatFree(void *ptr)
{
free(ptr);
}
/**
* Compress a file using Heatshrink
*
* @param[in] in - pointer to the uncompressed input
* @param insize - len of the uncompressed input
* @param[out] out - destination buffer for the compressed data
* @param outcap - capacity of the output buffer
* @param level - compression level, 1-9; -1 for default.
* @return actual length of the compressed data
*/
size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
{
const uint8_t *inp = in;
uint8_t *outp = out;
size_t len;
int ws[] = {5, 6, 8, 11, 13};
int ls[] = {3, 3, 4, 4, 4};
HSE_poll_res pres;
HSE_sink_res sres;
size_t r;
if (level == -1) { level = 8; }
level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4
heatshrink_encoder *enc = heatshrink_encoder_alloc(ws[level], ls[level]);
if (enc == NULL) {
perror("allocating mem for heatshrink");
exit(1);
}
//Save encoder parms as first byte
*outp = (ws[level] << 4) | ls[level];
outp++;
outcap--;
r = 1;
do {
if (insize > 0) {
sres = heatshrink_encoder_sink(enc, inp, insize, &len);
if (sres != HSER_SINK_OK) { break; }
inp += len;
insize -= len;
if (insize == 0) { heatshrink_encoder_finish(enc); }
}
do {
pres = heatshrink_encoder_poll(enc, outp, outcap, &len);
if (pres != HSER_POLL_MORE && pres != HSER_POLL_EMPTY) { break; }
outp += len;
outcap -= len;
r += len;
} while (pres == HSER_POLL_MORE);
} while (insize != 0);
if (insize != 0) {
fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", (int) insize, sres, pres);
exit(1);
}
heatshrink_encoder_free(enc);
return r;
}
/**
* Compress a file using Gzip
*
* @param[in] in - pointer to the uncompressed input
* @param insize - len of the uncompressed input
* @param[out] out - destination buffer for the compressed data
* @param outcap - capacity of the output buffer
* @param level - compression level, 1-9; -1 for default.
* @return actual length of the compressed data
*/
size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
{
z_stream stream;
int zresult;
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
stream.opaque = Z_NULL;
stream.next_in = in;
stream.avail_in = insize;
stream.next_out = out;
stream.avail_out = outcap;
// 31 -> 15 window bits + 16 for gzip
zresult = deflateInit2(&stream, level, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
if (zresult != Z_OK) {
fprintf(stderr, "DeflateInit2 failed with code %d\n", zresult);
exit(1);
}
zresult = deflate(&stream, Z_FINISH);
if (zresult != Z_STREAM_END) {
fprintf(stderr, "Deflate failed with code %d\n", zresult);
exit(1);
}
zresult = deflateEnd(&stream);
if (zresult != Z_OK) {
fprintf(stderr, "DeflateEnd failed with code %d\n", zresult);
exit(1);
}
return stream.total_out;
}
/**
* Check if a file name should be compressed by gzip
*
* @param name - file name
* @return true if should compress
*/
bool shouldCompressGzip(const char *name)
{
if (!s_gzipExtensions) { return false; }
if (s_gzipAll) { return true; }
const char *ext = name + strlen(name);
while (*ext != '.') {
ext--;
if (ext < name) {
// no dot in file name -> no extension -> nothing to match against
return false;
}
}
ext++;
int i = 0;
while (s_gzipExtensions[i] != NULL) {
if (strcasecmp(ext, s_gzipExtensions[i]) == 0) {
return true;
}
i++;
}
return false;
}
/**
* Parse a list of gzipped extensions
*
* @param input - list of comma-separated extensions, e.g. "jpg,png"
* @return
*/
void parseGzipExtensions(char *input)
{
char *token;
char *extList = input;
int count = 2; // one for first element, second for terminator
// count elements
while (*extList != 0) {
if (*extList == ',') { count++; }
extList++;
}
// split string
extList = input;
s_gzipExtensions = malloc(count * sizeof(char *));
count = 0;
token = strtok(extList, ",");
while (token) {
s_gzipExtensions[count++] = token;
token = strtok(NULL, ",");
}
// terminate list
s_gzipExtensions[count] = NULL;
}
/**
* Process a file.
*
* @param fd - filedes
* @param[in] name - filename to embed in the archive
* @param compression_mode - compression mode
* @param level - compression level for heatshrink, 1-9
* @param[out] compName - the used compression is output here (for debug print)
* @return - size of the output, in percent (100% = no compression)
*/
int handleFile(int fd, const char *name, int compression_mode, int level, const char **compName)
{
uint8_t *fdat = NULL, *cdat = NULL, *cdatbuf = NULL;
uint32_t size, csize;
EspFsHeader h;
uint16_t realNameLen;
uint8_t flags = 0;
size = lseek(fd, 0, SEEK_END);
fdat = malloc(size);
lseek(fd, 0, SEEK_SET);
read(fd, fdat, size);
if (shouldCompressGzip(name)) {
csize = size * 3;
if (csize < 100) { // gzip has some headers that do not fit when trying to compress small files
csize = 100;
} // enlarge buffer if this is the case
cdat = cdatbuf = malloc(csize);
csize = compressGzip(fdat, size, cdat, csize, level);
compression_mode = COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger
flags = FLAG_GZIP;
} else if (compression_mode == COMPRESS_NONE) {
csize = size;
cdat = fdat;
} else if (compression_mode == COMPRESS_HEATSHRINK) {
cdat = cdatbuf = malloc(size * 2);
csize = compressHeatshrink(fdat, size, cdat, size * 2, level);
} else {
fprintf(stderr, "Unknown compression - %d\n", compression_mode);
exit(1);
}
if (csize > size) {
fprintf(stderr, "! Compression enbiggened %s, embed as plain\n", name);
//Compressing enbiggened this file. Revert to uncompressed store.
compression_mode = COMPRESS_NONE;
csize = size;
cdat = fdat;
flags = 0;
}
//Fill header data
h.magic = htole32(ESPFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
h.flags = flags;
h.compression = (int8_t) compression_mode;
h.nameLen = realNameLen = strlen(name) + 1; // zero terminator
uint32_t padbytes = 0;
if (h.nameLen & 3) {
//Round to next 32bit boundary
padbytes = 4 - (h.nameLen & 3);
h.nameLen += padbytes; // include the bytes in "name" to make parsing easier - these will be zeroed out, so the c-string remains the same.
}
h.nameLen = htole16(h.nameLen);
h.fileLenComp = htole32(csize);
h.fileLenDecomp = htole32(size);
write(s_outFd, &h, sizeof(EspFsHeader));
write(s_outFd, name, realNameLen);
if (padbytes) {
write(s_outFd, "\0\0\0", padbytes); // these zeros are included in h.nameLen
}
write(s_outFd, cdat, csize);
//Pad out to 32bit boundary - the parser does this automatically when walking over the archive.
if (csize & 3) {
padbytes = 4 - (csize & 3);
write(s_outFd, "\0\0\0", padbytes);
csize += padbytes;
}
free(fdat);
if (cdatbuf) {
// free the buffer allocated for compression output
free(cdatbuf);
}
// debug outputs ...
if (compName != NULL) {
if (h.compression == COMPRESS_HEATSHRINK) {
*compName = "heatshrink";
} else if (h.compression == COMPRESS_NONE) {
if (h.flags & FLAG_GZIP) {
*compName = "gzip";
} else {
*compName = "none";
}
} else {
*compName = "unknown";
}
}
// get compression % (lower is better)
return size ? (int) ((csize * 100) / size) : 100;
}
/**
* Write final dummy header with FLAG_LASTFILE set.
*/
void finishArchive()
{
EspFsHeader h;
h.magic = htole32(ESPFS_MAGIC);
h.flags = FLAG_LASTFILE;
h.compression = COMPRESS_NONE;
h.nameLen = 0;
h.fileLenComp = 0;
h.fileLenDecomp = 0;
write(s_outFd, &h, sizeof(EspFsHeader));
}
/**
* Queue a file for adding to the archive.
* Appends it to the `s_inputFiles` linked list.
*
* @param name - file name to add
*/
void queueInputFile(const char *name)
{
fprintf(stderr, "INFILE: %s\n", name);
struct InputFileLinkedListEntry *tmp = malloc(sizeof(struct InputFileLinkedListEntry));
tmp->name = strdup(name);
tmp->next = NULL;
if (s_lastInputFile == NULL) {
s_inputFiles = tmp;
s_lastInputFile = tmp;
} else {
s_lastInputFile->next = tmp;
s_lastInputFile = tmp;
}
}
int main(int argc, char **argv)
{
int f;
char inputFileName[1024];
char *realName;
struct stat statBuf;
int serr;
int rate;
int err = 0;
int compType; //default compression type - heatshrink
int compLvl = -1;
bool use_gzip = false;
compType = COMPRESS_HEATSHRINK;
int c;
char *outfile = NULL;
char *parseFile = NULL;
char *stripPath = NULL;
while (1) {
int option_index = 0;
static struct option long_options[] = {
{"parse", required_argument, 0, 'p'},
{"compress", required_argument, 0, 'c'},
{"gzip", no_argument, 0, 'z'},
{"gzip-all", no_argument, 0, 'G'},
{"level", required_argument, 0, 'l'},
{"gzip-exts", required_argument, 0, 'g'},
{"input", required_argument, 0, 'i'},
{"output", required_argument, 0, 'o'},
{"strip-path", required_argument, 0, 'S'},
{"help", no_argument, 0, 'h'},
{0, 0, 0, 0}
};
c = getopt_long(argc, argv, "c:l:g:zGhp:i:o:0123456789",
long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'h':
goto show_help;
case 'z':
use_gzip = true;
break;
case '0' ... '9':
compLvl = c - '0';
break;
case 'p':
parseFile = strdup(optarg);
break;
case 'S':
stripPath = strdup(optarg);
break;
case 'c':
compType = atoi(optarg);
break;
case 'G':
use_gzip = true;
s_gzipAll = true;
break;
case 'g':
use_gzip = true;
parseGzipExtensions(optarg);
break;
case 'l':
compLvl = atoi(optarg);
if (compLvl < 1 || compLvl > 9) {
fprintf(stderr, "Bad compression level: %d\n", compLvl);
err = 1;
goto show_help;
}
break;
case 'i':
queueInputFile(optarg);
break;
case 'o':
outfile = strdup(optarg);
break;
case '?':
goto show_help;
default:
fprintf(stderr, "Unknown option: %c\n", c);
err = 1;
goto show_help;
}
}
if (parseFile) {
parseEspfsFileAndShowItsContents(parseFile);
exit(0);
}
if (s_gzipExtensions == NULL && use_gzip) {
parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS));
}
if (optind < argc) {
while (optind < argc) {
queueInputFile(argv[optind++]);
}
}
if (!s_inputFiles) {
fprintf(stderr, "Reading input file names from stdin\n");
while (fgets(inputFileName, sizeof(inputFileName), stdin)) {
//Kill off '\n' at the end
inputFileName[strlen(inputFileName) - 1] = 0;
queueInputFile(inputFileName);
}
}
FILE *outfp = NULL;
if (outfile) {
fprintf(stderr, "Writing to %s\n", outfile);
outfp = fopen(outfile, "w+");
if (!outfp) {
perror(outfile);
return 1;
}
s_outFd = fileno(outfp);
ftruncate(s_outFd, 0);
} else {
fprintf(stderr, "Writing to stdout\n\n");
}
struct InputFileLinkedListEntry *entry = s_inputFiles;
while (entry) {
char *name = entry->name;
//Only include files
serr = stat(name, &statBuf);
if ((serr == 0) && S_ISREG(statBuf.st_mode)) {
//Strip off './' or '/' madness.
char *embeddedName = name;
f = open(name, O_RDONLY);
// relative path starting with ./, remove that
if (embeddedName[0] == '.' && embeddedName[1] == '/') {
embeddedName += 2;
}
// remove prefix
if (stripPath && 0 == strncmp(embeddedName, stripPath, strlen(stripPath))) {
embeddedName += strlen(stripPath);
}
// remove leading slash, if any
if (embeddedName[0] == '/') { embeddedName++; }
if (f > 0) {
const char *compName = "unknown";
rate = handleFile(f, embeddedName, compType, compLvl, &compName);
fprintf(stderr, "%s (%d%%, %s)\n", embeddedName, rate, compName);
close(f);
} else {
perror(name);
}
} else if (serr != 0) {
perror(name);
}
entry = entry->next;
}
finishArchive();
fsync(s_outFd);
if (outfp) {
fclose(outfp);
}
return 0;
show_help:
fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
fprintf(stderr, "Options:\n");
fprintf(stderr, "[-p|--parse FILE]\n Parse an espfs file and show a list of its contents. No other options apply in this mode.\n");
fprintf(stderr, "[-c|--compress COMPRESSOR]\n 0 - None, 1 - Heatshrink (default)\n");
fprintf(stderr, "[-l|--level LEVEL] or [-0 through -9]\n compression level 1-9, higher is better but uses more RAM\n");
fprintf(stderr, "[-z|--gzip]\n use gzip for files with extensions matching "DEFAULT_GZIP_EXTS"\n");
fprintf(stderr, "[-G|--gzip-all]\n use gzip for all files\n");
fprintf(stderr, "[-g|--gzip-exts GZIPPED_EXTENSIONS]\n use gzip for files with custom extensions, comma-separated\n");
fprintf(stderr, "[-i|--input FILE]\n Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin if not specified by args\n");
fprintf(stderr, "[-o|--output FILE]\n Output file name; if not specified, outputs to stdout\n");
fprintf(stderr, "[-h|--help\n Show help.\n\n");
exit(err);
}