SpriteHTTPD - embedded HTTP server with read-only filesystem and templating, originally developed for ESP8266, now stand-alone and POSIX compatible.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
spritehttpd/fstool/main.c

784 lines
25 KiB

#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <zlib.h>
#include <getopt.h>
#include <stdbool.h>
#include <errno.h>
#include "espfsformat.h"
#include "heatshrink_encoder.h"
#include "parsing.h"
#include "httpd-logging.h"
void show_version(char **argv);
void show_help(int retval, char **argv);
#define DEFAULT_GZIP_EXTS "css,js,svg,png,jpg,jpeg,webm,ico,gif"
#define DEFAULT_C_VARNAME "espfs_image"
struct InputFileLinkedListEntry {
char *name;
struct InputFileLinkedListEntry *next;
};
/// two ends of a linked list with input files
static struct InputFileLinkedListEntry *s_inputFiles = NULL;
static struct InputFileLinkedListEntry *s_lastInputFile = NULL;
/// Output file FD
static int s_outFd = STDOUT_FILENO;
/// Array of gzipped extensions, ends with a NULL pointer
static char **s_gzipExtensions = NULL;
/// Gzip all files
static bool s_gzipAll = false;
// impls to satisfy defs in the config header
void *httpdPlatMalloc(size_t len)
{
return malloc(len);
}
void httpdPlatFree(void *ptr)
{
free(ptr);
}
/**
* Compress a file using Heatshrink
*
* @param[in] in - pointer to the uncompressed input
* @param insize - len of the uncompressed input
* @param[out] out - destination buffer for the compressed data
* @param outcap - capacity of the output buffer
* @param level - compression level, 1-9; -1 for default.
* @return actual length of the compressed data
*/
size_t compressHeatshrink(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap, int level)
{
const uint8_t *inp = in;
uint8_t *outp = out;
size_t len;
uint8_t ws[] = {5, 6, 8, 11, 13};
uint8_t ls[] = {3, 3, 4, 4, 4};
HSE_poll_res pres = 0;
HSE_sink_res sres = 0;
size_t r;
if (level == -1) { level = 8; }
level = (level - 1) / 2; //level is now 0, 1, 2, 3, 4
heatshrink_encoder *enc = heatshrink_encoder_alloc(ws[level], ls[level]);
if (enc == NULL) {
espfs_error("allocating mem for heatshrink");
exit(1);
}
//Save encoder parms as first byte
*outp = (uint8_t) ((ws[level] << 4) | ls[level]);
outp++;
outcap--;
r = 1;
do {
if (insize > 0) {
sres = heatshrink_encoder_sink(enc, inp, insize, &len);
if (sres != HSER_SINK_OK) { break; }
inp += len;
insize -= len;
if (insize == 0) { heatshrink_encoder_finish(enc); }
}
do {
pres = heatshrink_encoder_poll(enc, outp, outcap, &len);
if (pres != HSER_POLL_MORE && pres != HSER_POLL_EMPTY) { break; }
outp += len;
outcap -= len;
r += len;
} while (pres == HSER_POLL_MORE);
} while (insize != 0);
if (insize != 0) {
espfs_error("Heatshrink: Bug? insize is still %d. sres=%d pres=%d", (int) insize, sres, pres);
exit(1);
}
heatshrink_encoder_free(enc);
return r;
}
/**
* Compress a file using Gzip
*
* @param[in] in - pointer to the uncompressed input
* @param insize - len of the uncompressed input
* @param[out] out - destination buffer for the compressed data
* @param outcap - capacity of the output buffer
* @return actual length of the compressed data
*/
size_t compressGzip(const uint8_t *in, size_t insize, uint8_t *out, size_t outcap)
{
z_stream stream;
int zresult;
stream.zalloc = Z_NULL;
stream.zfree = Z_NULL;
stream.opaque = Z_NULL;
stream.next_in = in;
stream.avail_in = (uInt) insize;
stream.next_out = out;
stream.avail_out = (uInt) outcap;
// 31 -> 15 window bits + 16 for gzip
zresult = deflateInit2(&stream, Z_BEST_COMPRESSION /* we want the smallest possible files */, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY);
if (zresult != Z_OK) {
espfs_error("DeflateInit2 failed with code %d", zresult);
exit(1);
}
zresult = deflate(&stream, Z_FINISH);
if (zresult != Z_STREAM_END) {
espfs_error("Deflate failed with code %d", zresult);
exit(1);
}
zresult = deflateEnd(&stream);
if (zresult != Z_OK) {
espfs_error("DeflateEnd failed with code %d", zresult);
exit(1);
}
return stream.total_out;
}
/**
* Check if a file name should be compressed by gzip
*
* @param name - file name
* @return true if should compress
*/
bool shouldCompressGzip(const char *name)
{
if (!s_gzipExtensions) { return false; }
if (s_gzipAll) { return true; }
const char *ext = name + strlen(name);
while (*ext != '.') {
ext--;
if (ext < name) {
// no dot in file name -> no extension -> nothing to match against
return false;
}
}
ext++;
int i = 0;
while (s_gzipExtensions[i] != NULL) {
if (strcasecmp(ext, s_gzipExtensions[i]) == 0) {
return true;
}
i++;
}
return false;
}
/**
* Parse a list of gzipped extensions
*
* @param input - list of comma-separated extensions, e.g. "jpg,png"
* @return
*/
void parseGzipExtensions(char *input)
{
char *token;
char *extList = input;
size_t count = 2; // one for first element, second for terminator
// count elements
while (*extList != 0) {
if (*extList == ',') { count++; }
extList++;
}
// split string
extList = input;
s_gzipExtensions = malloc(count * sizeof(char *));
count = 0;
token = strtok(extList, ",");
while (token) {
s_gzipExtensions[count++] = token;
token = strtok(NULL, ",");
}
// terminate list
s_gzipExtensions[count] = NULL;
}
/**
* Process a file.
*
* @param fd - filedes
* @param[in] name - filename to embed in the archive
* @param compression_mode - compression mode
* @param level - compression level for heatshrink, 1-9
* @param[out] compName - the used compression is output here (for debug print)
* @return - size of the output, in percent (100% = no compression)
*/
int handleFile(int fd, const char *name, int compression_mode, int level, const char **compName)
{
uint8_t *fdat = NULL, *cdat = NULL, *cdatbuf = NULL;
size_t size, csize;
EspFsHeader h;
uint16_t realNameLen;
uint8_t flags = 0;
size = (size_t) lseek(fd, 0, SEEK_END);
fdat = malloc(size);
lseek(fd, 0, SEEK_SET);
read(fd, fdat, size);
if (shouldCompressGzip(name)) {
csize = size * 3;
if (csize < 100) { // gzip has some headers that do not fit when trying to compress small files
csize = 100;
} // enlarge buffer if this is the case
cdat = cdatbuf = malloc(csize);
csize = compressGzip(fdat, size, cdat, csize);
compression_mode = EFS_COMPRESS_NONE; // don't use heatshrink if gzip was already used - it would only make it bigger
flags = EFS_FLAG_GZIP;
} else if (compression_mode == EFS_COMPRESS_NONE) {
csize = size;
cdat = fdat;
} else if (compression_mode == EFS_COMPRESS_HEATSHRINK) {
cdat = cdatbuf = malloc(size * 2);
csize = compressHeatshrink(fdat, size, cdat, size * 2, level);
} else {
espfs_error("Unknown compression - %d", compression_mode);
exit(1);
}
if (csize > size) {
espfs_dbg("! Compression enbiggened %s, embed as plain", name);
//Compressing enbiggened this file. Revert to uncompressed store.
compression_mode = EFS_COMPRESS_NONE;
csize = size;
cdat = fdat;
flags = 0;
}
//Fill header data
h.magic = htole32(EFS_MAGIC); // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
h.flags = flags;
h.compression = (uint8_t) compression_mode;
h.nameLen = realNameLen = (uint16_t) strlen(name) + 1; // zero terminator
uint16_t padbytes = 0;
if (h.nameLen & 3) {
//Round to next 32bit boundary
padbytes = 4 - (h.nameLen & 3);
h.nameLen += padbytes; // include the bytes in "name" to make parsing easier - these will be zeroed out, so the c-string remains the same.
}
h.nameLen = htole16(h.nameLen);
h.fileLenComp = htole32((uint32_t) csize);
h.fileLenDecomp = htole32((uint32_t) size);
write(s_outFd, &h, sizeof(EspFsHeader));
write(s_outFd, name, realNameLen);
if (padbytes) {
write(s_outFd, "\0\0\0", padbytes); // these zeros are included in h.nameLen
}
write(s_outFd, cdat, csize);
//Pad out to 32bit boundary - the parser does this automatically when walking over the archive.
if (csize & 3) {
padbytes = 4 - (csize & 3);
write(s_outFd, "\0\0\0", padbytes);
csize += padbytes;
}
free(fdat);
if (cdatbuf) {
// free the buffer allocated for compression output
free(cdatbuf);
}
// debug outputs ...
if (compName != NULL) {
if (h.compression == EFS_COMPRESS_HEATSHRINK) {
*compName = "heatshrink";
} else if (h.compression == EFS_COMPRESS_NONE) {
if (h.flags & EFS_FLAG_GZIP) {
*compName = "gzip";
} else {
*compName = "none";
}
} else {
*compName = "unknown";
}
}
// get compression % (lower is better)
return size ? (int) ((csize * 100) / size) : 100;
}
/**
* Write final dummy header with FLAG_LASTFILE set.
*/
void finishArchive(void)
{
EspFsHeader h;
h.magic = htole32(EFS_MAGIC);
h.flags = EFS_FLAG_LASTFILE;
h.compression = EFS_COMPRESS_NONE;
h.nameLen = 0;
h.fileLenComp = 0;
h.fileLenDecomp = 0;
write(s_outFd, &h, sizeof(EspFsHeader));
}
/**
* Queue a file for adding to the archive.
* Appends it to the `s_inputFiles` linked list.
*
* @param name - file name to add
*/
void queueInputFile(const char *name)
{
espfs_dbg("INFILE: %s", name);
struct InputFileLinkedListEntry *tmp = malloc(sizeof(struct InputFileLinkedListEntry));
tmp->name = strdup(name);
tmp->next = NULL;
if (s_lastInputFile == NULL) {
s_inputFiles = tmp;
s_lastInputFile = tmp;
} else {
s_lastInputFile->next = tmp;
s_lastInputFile = tmp;
}
}
enum OpMode {
OM_INVALID = 0,
OM_PACK = 'P',
OM_LIST = 'L',
OM_EXTRACT = 'X',
OM_EMBED = 'M',
};
#define BUFLEN 1024
int main(const int argc, char **argv)
{
int f;
char inputFileName[BUFLEN];
char tempbuf[BUFLEN];
struct stat statBuf;
int serr;
int rate;
int compType; //default compression type - heatshrink
int compLvl = -1;
bool use_gzip = false;
enum OpMode opmode = OM_INVALID;
compType = EFS_COMPRESS_HEATSHRINK;
int c;
char *outFileName = NULL;
char *c_varname = NULL;
char *stripPath = NULL;
char *extractFileName = NULL;
size_t num_input_files = 0;
bool read_from_stdin = false;
while (1) {
int option_index = 0;
static struct option long_options[] = {
// Help
{"help", no_argument, 0, 'h'},
{"version", no_argument, 0, 'V'},
// Main operation (one at a time)
{"pack", no_argument, 0, 'P'},
{"list", no_argument, 0, 'L'},
{"extract", required_argument, 0, 'X'},
{"embed", no_argument, 0, 'M'},
// Common options
{"input", required_argument, 0, 'i'}, // input file; "-" to read them as lines on stdin; can be repeated in case of --pack.
// Input files can also be given as stand-alone arguments at the end, e.g. as a result of glob
{"output", required_argument, 0, 'o'}, // output file; "-" for stdout (default)
// Options for --pack
{"compress", required_argument, 0, 'c'}, // 0 = no, 1 = heatshrink (def)
{"level", required_argument, 0, 'l'}, // Heatshrink compression level 1-9, -1 = default
{"gzip", optional_argument, 0, 'z'}, // Gzipped extensions; no arg = default, "*" = all, comma-separated list = custom exts
{"strip", required_argument, 0, 's'}, // path removed from all input files, e.g. when they are in a subfolder
// Options for --embed
{"varname", required_argument, 0, 'n'}, // name of the array; the length variable is {varname}_len
{ /* end marker */ }
};
c = getopt_long(argc, argv, "h?VPLX:Mi:o:c:l:z::s:n:",
long_options, &option_index);
if (c == -1) {
break;
}
switch (c) {
case 'h':
case '?':
show_help(0, argv);
case 'V':
show_version(argv);
case 'P':
opmode = OM_PACK;
break;
case 'L':
opmode = OM_LIST;
break;
case 'X':
opmode = OM_EXTRACT;
if (extractFileName) {
espfs_error("can extract only one file at a time!");
exit(1);
}
extractFileName = strdup(optarg);
break;
case 'M':
opmode = OM_EMBED;
break;
case 'i':
if (0 == strcmp(optarg, "-")) {
read_from_stdin = true;
} else {
queueInputFile(optarg);
}
num_input_files++;
break;
case 'o':
outFileName = strdup(optarg);
break;
case 'c':
errno = 0;
compType = (int) strtol(optarg, NULL, 10);
if (errno != 0 || compType < 0 || compType > 1) {
espfs_error("Bad compression mode: %s", optarg);
exit(1);
}
break;
case 'l':
errno = 0;
compLvl = (int) strtol(optarg, NULL, 10);
if (errno != 0 || compLvl < 1 || compLvl > 9) {
espfs_error("Bad compression level: %s", optarg);
exit(1);
}
break;
case 'z':
use_gzip = true;
if (optarg) {
if (0 == strcmp("*", optarg)) {
s_gzipAll = true;
} else {
parseGzipExtensions(optarg);
}
} else {
parseGzipExtensions(strdup(DEFAULT_GZIP_EXTS)); // memory leak! ehh
}
break;
case 's':
stripPath = strdup(optarg);
break;
case 'n':
c_varname = strdup(optarg);
break;
default:
espfs_error("Unknown option: %c", c);
exit(1);
}
}
if (!use_gzip) {
s_gzipExtensions = NULL;
}
bool want_output;
bool allows_multiple_inputs;
switch (opmode) {
case OM_PACK:
want_output = true;
allows_multiple_inputs = true;
break;
case OM_LIST:
want_output = false;
allows_multiple_inputs = false;
break;
case OM_EXTRACT:
case OM_EMBED:
want_output = true;
allows_multiple_inputs = false;
break;
default:
espfs_error("Specify one of the operation modes: -P, -L, -X, -M");
exit(1);
}
/* Input */
while (optind < argc) {
char *s = argv[optind++];
if (0 == strcmp(s, "-")) {
read_from_stdin = true;
} else {
queueInputFile(s);
num_input_files++;
}
}
if (num_input_files == 0 && read_from_stdin && opmode == OM_PACK) {
read_from_stdin = false;
espfs_dbg("Reading input file names from stdin");
while (fgets(inputFileName, sizeof(inputFileName), stdin)) {
//Kill off '\n' at the end
inputFileName[strlen(inputFileName) - 1] = 0;
queueInputFile(inputFileName);
num_input_files++;
}
}
if (!read_from_stdin) {
if (num_input_files == 0) {
if (allows_multiple_inputs) {
espfs_error("Specify input file(s)!");
} else {
espfs_error("Specify input file!");
}
exit(1);
} else if (!allows_multiple_inputs && num_input_files > 1) {
espfs_error("Mode %c requires exactly one input file!", opmode);
exit(1);
}
}
char *inFileName = read_from_stdin ? NULL : s_inputFiles->name;
/* Output */
if (!want_output && outFileName) {
espfs_error("Output file is not allowed in %c mode!", opmode);
exit(1);
}
FILE *outFile = NULL;
bool write_to_stdout = outFileName && (0 == strcmp("-", outFileName));
if (outFileName && !write_to_stdout) {
espfs_dbg("Writing to %s", outFileName);
outFile = fopen(outFileName, "w+");
if (!outFile) {
perror(outFileName);
return 1;
}
s_outFd = fileno(outFile);
ftruncate(s_outFd, 0);
} else {
if (outFileName) {
free(outFileName);
outFileName = NULL;
}
if (want_output && !write_to_stdout) {
espfs_error("Specify output file! Use -o - for stdout");
exit(1);
}
}
/* Do it! */
switch (opmode) {
case OM_PACK: {
struct InputFileLinkedListEntry *entry = s_inputFiles;
while (entry) {
char *name = entry->name;
//Only include files
serr = stat(name, &statBuf);
if ((serr == 0) && S_ISREG(statBuf.st_mode)) {
//Strip off './' or '/' madness.
char *embeddedName = name;
f = open(name, O_RDONLY);
if (f > 0) {
// relative path starting with ./, remove that
if (embeddedName[0] == '.' && embeddedName[1] == '/') {
embeddedName += 2;
}
// remove prefix
if (stripPath && 0 == strncmp(embeddedName, stripPath, strlen(stripPath))) {
embeddedName += strlen(stripPath);
}
// remove leading slash, if any
if (embeddedName[0] == '/') { embeddedName++; }
const char *compName = "unknown";
rate = handleFile(f, embeddedName, compType, compLvl, &compName);
(void) rate;
espfs_dbg("%s (%d%%, %s)", embeddedName, rate, compName);
close(f);
} else {
snprintf(tempbuf, BUFLEN, "Open %s for reading: %s", name, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
} else if (serr != 0) {
snprintf(tempbuf, BUFLEN, "Stat %s: %s", name, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
entry = entry->next;
}
finishArchive();
fsync(s_outFd);
if (outFile) {
fclose(outFile);
}
} break;
case OM_LIST: {
if (!inFileName) {
espfs_error("Input file required!");
exit(1);
}
parseEspfsImage(inFileName, NULL, s_outFd);
} break;
case OM_EXTRACT: {
if (!inFileName) {
espfs_error("Input file required!");
exit(1);
}
parseEspfsImage(inFileName, extractFileName, s_outFd);
} break;
case OM_EMBED: {
FILE *inFile = NULL;
int inFD = STDIN_FILENO;
if (!read_from_stdin) {
inFile = fopen(inFileName, "r");
if (!inFile) {
snprintf(tempbuf, BUFLEN, "Open %s for reading: %s", inFileName, strerror(errno));
espfs_error("%s", tempbuf);
exit(1);
}
inFD = fileno(inFile);
}
if (!c_varname) {
c_varname = strdup(DEFAULT_C_VARNAME);
}
size_t len;
len = (size_t) snprintf(tempbuf, BUFLEN, "unsigned char %s[] = {", c_varname);
write(s_outFd, tempbuf, len);
uint8_t u;
size_t imageLen = 0;
while (1 == read(inFD, &u, 1)) {
len = (size_t) snprintf(tempbuf, BUFLEN, "%s0x%02x,", ((imageLen % 16) ? " " : "\n "), u);
write(s_outFd, tempbuf, len);
imageLen++;
}
len = (size_t) snprintf(tempbuf, BUFLEN, "\n};\nunsigned int %s_len = %lu;\n", c_varname, imageLen);
write(s_outFd, tempbuf, len);
fsync(s_outFd);
if (outFile) {
fclose(outFile);
}
if (inFile) {
fclose(inFile);
}
} break;
default:
__builtin_unreachable();
}
return 0;
}
__attribute__((noreturn))
void show_version(char **argv)
{
// to stdout
printf("%s #%s\n", argv[0], GIT_HASH);
exit(0);
}
__attribute__((noreturn))
void show_help(int retval, char **argv)
{
// to stderr, this can happen as a response to bad args
// ##########**********##########**********##########----------$$$$$$$$$$----------$$$$$$$$$$----------| 80 chars
fprintf(stderr, "%s #%s - Program to create and parse espfs images\n", argv[0], GIT_HASH);
fprintf(stderr, "\n"
"One main operation mode must be selected:\n"
" -P, --pack Create a binary fs image\n"
" -L, --list Read a binary fs image and show the contents\n"
" -X, --extract=NAME Read a binary fs image and extract a file with the given name\n"
" -M, --embed Read a binary file (typically the binary fs image produced by -P) and\n"
" convert it to C syntax byte array and a constant with its length.\n"
" -h, -?, --help Show this help (has precedence over other options)\n"
"\n"
"Additional arguments specify the input, output and parameters:\n"
" -i, --input=FILE Input file name. Can be used multiple times. For convenience with globs,\n"
" input files can be given at the end of the option string without -i.\n"
" The \"-\" filename means \"read from stdin\":\n"
" - pack: reads file names to pack from stdin (e.g. piped from `find`)\n"
" - embed: read the binary file from stdin (e.g. piped from the --pack\n"
" option, avoiding the creation of a temporary binary file)\n"
" Stdin reading is not supported for options --extract and --list.\n"
"\n"
" -o, --output=FILE Output file, \"-\" for stdout.\n"
" Output can't be changed in --list mode.\n"
"\n"
"Pack options:\n"
" -c, --compress=MODE Compression mode, 0=none, 1=heatshrink (default). Not used if the file\n"
" is gzipped - additional compression wouldn't be useful.\n"
" -l, --level=LEVEL Compression level, 1-9, -1=default. 1=worst, 9=best compression, but uses\n"
" more RAM to unpack.\n"
" -s, --strip=PATH Strip a path prefix from all packed files (e.g. a subfolder name)\n"
" -z, --gzip[=EXTS] Enable gzip compression for some file types (filename extensions).\n"
" By default, enabled for "DEFAULT_GZIP_EXTS".\n"
" Set EXTS to * to enable gzip for all files. Custom extensions are set as\n"
" a comma-separated list.\n"
"\n"
"Embed options:\n"
" -n, --varname=VARNAME Set custom array name to use in the generated C code. The length constant\n"
" is called {VARNAME}_len. The default VARNAME is \""DEFAULT_C_VARNAME"\"\n"
);
exit(retval);
}