@ -11,12 +11,10 @@
# include "espfsformat.h" # include "espfsformat.h"
# include "heatshrink_encoder.h" # include "heatshrink_encoder.h"
# include "espfs .h" # include "parsing .h"
# define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif" # define DEFAULT_GZIP_EXTS "html,css,js,svg,png,jpg,gif"
// static variables
static int s_outFd = 1 ;
struct InputFileLinkedListEntry ; struct InputFileLinkedListEntry ;
struct InputFileLinkedListEntry { struct InputFileLinkedListEntry {
@ -24,10 +22,16 @@ struct InputFileLinkedListEntry {
struct InputFileLinkedListEntry * next ; struct InputFileLinkedListEntry * next ;
} ; } ;
/// two ends of a linked list with input files
static struct InputFileLinkedListEntry * s_inputFiles = NULL ; static struct InputFileLinkedListEntry * s_inputFiles = NULL ;
static struct InputFileLinkedListEntry * s_lastInputFile = NULL ; static struct InputFileLinkedListEntry * s_lastInputFile = NULL ;
/// Output file FD
static int s_outFd = 1 ;
/// Array of gzipped extensions, ends with a NULL pointer
static char * * s_gzipExtensions = NULL ; static char * * s_gzipExtensions = NULL ;
/// Gzip all files
static bool s_gzipAll = false ; static bool s_gzipAll = false ;
// impls to satisfy defs in the config header
// impls to satisfy defs in the config header
@ -41,9 +45,19 @@ void httpdPlatFree(void *ptr)
free ( ptr ) ; free ( ptr ) ;
} }
size_t compressHeatshrink ( uint8_t * in , size_t insize , uint8_t * out , size_t outcap , int level ) /**
* Compress a file using Heatshrink
*
* @ param [ in ] in - pointer to the uncompressed input
* @ param insize - len of the uncompressed input
* @ param [ out ] out - destination buffer for the compressed data
* @ param outcap - capacity of the output buffer
* @ param level - compression level , 1 - 9 ; - 1 for default .
* @ return actual length of the compressed data
*/
size_t compressHeatshrink ( const uint8_t * in , size_t insize , uint8_t * out , size_t outcap , int level )
{ {
uint8_t * inp = in ; const uint8_t * inp = in ;
uint8_t * outp = out ; uint8_t * outp = out ;
size_t len ; size_t len ;
int ws [ ] = { 5 , 6 , 8 , 11 , 13 } ; int ws [ ] = { 5 , 6 , 8 , 11 , 13 } ;
@ -82,7 +96,7 @@ size_t compressHeatshrink(uint8_t *in, size_t insize, uint8_t *out, size_t outca
} while ( insize ! = 0 ) ; } while ( insize ! = 0 ) ;
if ( insize ! = 0 ) { if ( insize ! = 0 ) {
fprintf ( stderr , " Heatshrink: Bug? insize is still %d. sres=%d pres=%d \n " , ( int ) insize , sres , pres ) ; fprintf ( stderr , " Heatshrink: Bug? insize is still %d. sres=%d pres=%d \n " , ( int ) insize , sres , pres ) ;
exit ( 1 ) ; exit ( 1 ) ;
} }
@ -90,7 +104,17 @@ size_t compressHeatshrink(uint8_t *in, size_t insize, uint8_t *out, size_t outca
return r ; return r ;
} }
size_t compressGzip ( uint8_t * in , size_t insize , uint8_t * out , size_t outsize , int level ) /**
* Compress a file using Gzip
*
* @ param [ in ] in - pointer to the uncompressed input
* @ param insize - len of the uncompressed input
* @ param [ out ] out - destination buffer for the compressed data
* @ param outcap - capacity of the output buffer
* @ param level - compression level , 1 - 9 ; - 1 for default .
* @ return actual length of the compressed data
*/
size_t compressGzip ( const uint8_t * in , size_t insize , uint8_t * out , size_t outcap , int level )
{ {
z_stream stream ; z_stream stream ;
int zresult ; int zresult ;
@ -101,7 +125,7 @@ size_t compressGzip(uint8_t *in, size_t insize, uint8_t *out, size_t outsize, in
stream . next_in = in ; stream . next_in = in ;
stream . avail_in = insize ; stream . avail_in = insize ;
stream . next_out = out ; stream . next_out = out ;
stream . avail_out = outsize ; stream . avail_out = outcap ;
// 31 -> 15 window bits + 16 for gzip
// 31 -> 15 window bits + 16 for gzip
zresult = deflateInit2 ( & stream , level , Z_DEFLATED , 31 , 8 , Z_DEFAULT_STRATEGY ) ; zresult = deflateInit2 ( & stream , level , Z_DEFLATED , 31 , 8 , Z_DEFAULT_STRATEGY ) ;
if ( zresult ! = Z_OK ) { if ( zresult ! = Z_OK ) {
@ -124,6 +148,12 @@ size_t compressGzip(uint8_t *in, size_t insize, uint8_t *out, size_t outsize, in
return stream . total_out ; return stream . total_out ;
} }
/**
* Check if a file name should be compressed by gzip
*
* @ param name - file name
* @ return true if should compress
*/
bool shouldCompressGzip ( const char * name ) bool shouldCompressGzip ( const char * name )
{ {
if ( ! s_gzipExtensions ) { return false ; } if ( ! s_gzipExtensions ) { return false ; }
@ -150,7 +180,13 @@ bool shouldCompressGzip(const char *name)
return false ; return false ;
} }
int parseGzipExtensions ( char * input ) /**
* Parse a list of gzipped extensions
*
* @ param input - list of comma - separated extensions , e . g . " jpg,png "
* @ return
*/
void parseGzipExtensions ( char * input )
{ {
char * token ; char * token ;
char * extList = input ; char * extList = input ;
@ -173,19 +209,17 @@ int parseGzipExtensions(char *input)
} }
// terminate list
// terminate list
s_gzipExtensions [ count ] = NULL ; s_gzipExtensions [ count ] = NULL ;
return 1 ;
} }
/**
/**
* Process a file . * Process a file .
* *
* @ param fd - filedes * @ param fd - filedes
* @ param name - filename to embed in the archive * @ param [ in ] name - filename to embed in the archive
* @ param compression_mode - compression mode * @ param compression_mode - compression mode
* @ param level - compression level for heatshrink , 1 - 9 * @ param level - compression level for heatshrink , 1 - 9
* @ param [ out ] compName - the used compression is output here ( for debug print ) * @ param [ out ] compName - the used compression is output here ( for debug print )
* @ return * @ return - size of the output , in percent ( 100 % = no compression )
*/ */
int handleFile ( int fd , const char * name , int compression_mode , int level , const char * * compName ) int handleFile ( int fd , const char * name , int compression_mode , int level , const char * * compName )
{ {
@ -282,11 +316,13 @@ int handleFile(int fd, const char *name, int compression_mode, int level, const
return size ? ( int ) ( ( csize * 100 ) / size ) : 100 ; return size ? ( int ) ( ( csize * 100 ) / size ) : 100 ;
} }
//Write final dummy header with FLAG_LASTFILE set.
/**
* Write final dummy header with FLAG_LASTFILE set .
*/
void finishArchive ( ) void finishArchive ( )
{ {
EspFsHeader h ; EspFsHeader h ;
h . magic = htole32 ( ESPFS_MAGIC ) ; // ('E' << 0) + ('S' << 8) + ('f' << 16) + ('s' << 24);
h . magic = htole32 ( ESPFS_MAGIC ) ;
h . flags = FLAG_LASTFILE ; h . flags = FLAG_LASTFILE ;
h . compression = COMPRESS_NONE ; h . compression = COMPRESS_NONE ;
h . nameLen = 0 ; h . nameLen = 0 ;
@ -295,62 +331,13 @@ void finishArchive()
write ( s_outFd , & h , sizeof ( EspFsHeader ) ) ; write ( s_outFd , & h , sizeof ( EspFsHeader ) ) ;
} }
static size_t espfs_parse_filesize = - 1 ; /**
static int espfs_parse_fd = - 1 ; * Queue a file for adding to the archive .
* Appends it to the ` s_inputFiles ` linked list .
void parseEspfsFileAndShowItsContents ( const char * filename ) *
{ * @ param name - file name to add
int rv ; */
fprintf ( stderr , " Parsing: %s \n " , filename ) ; void queueInputFile ( const char * name )
FILE * f = fopen ( filename , " r " ) ;
if ( ! f ) {
perror ( filename ) ;
exit ( 1 ) ;
}
int fd = fileno ( f ) ;
espfs_parse_filesize = lseek ( fd , 0 , SEEK_END ) ;
lseek ( fd , 0 , SEEK_SET ) ;
espfs_parse_fd = fd ;
rv = espFsInit ( ) ;
if ( rv ! = 0 ) {
fprintf ( stderr , " Fail to init FS \n " ) ;
exit ( 1 ) ;
}
EspFsWalk walk ;
espFsWalkInit ( & walk ) ;
EspFsHeader header ;
uint32_t offset ;
char namebuf [ 1024 ] ;
while ( espFsWalkNext ( & walk , & header , namebuf , 1024 , & offset ) ) {
fprintf ( stderr , " at %04x: \" %s \" , flags: %02x, comp: %s, compLen: %d, plainLen: %d \n " , offset , namebuf , header . flags ,
header . compression = = 1 ? " HS " : " None " , header . fileLenComp , header . fileLenDecomp ) ;
}
fclose ( f ) ;
}
int httpdPlatEspfsRead ( void * dest , uint32_t offset , size_t len )
{
fprintf ( stderr , " FS read @ %d, len %d \n " , offset , len ) ;
if ( offset + len > espfs_parse_filesize ) {
fprintf ( stderr , " Read out fo range! \n " ) ;
return - 1 ;
}
lseek ( espfs_parse_fd , offset , SEEK_SET ) ;
read ( espfs_parse_fd , dest , len ) ;
return 0 ;
}
void queueInputFile ( char * name )
{ {
fprintf ( stderr , " INFILE: %s \n " , name ) ; fprintf ( stderr , " INFILE: %s \n " , name ) ;
@ -385,20 +372,22 @@ int main(int argc, char **argv)
int c ; int c ;
char * outfile = NULL ; char * outfile = NULL ;
char * parseFile = NULL ; char * parseFile = NULL ;
char * stripPath = NULL ;
while ( 1 ) { while ( 1 ) {
int option_index = 0 ; int option_index = 0 ;
static struct option long_options [ ] = { static struct option long_options [ ] = {
{ " parse " , required_argument , 0 , ' p ' } , { " parse " , required_argument , 0 , ' p ' } ,
{ " compress " , required_argument , 0 , ' c ' } , { " compress " , required_argument , 0 , ' c ' } ,
{ " gzip " , no_argument , 0 , ' z ' } , { " gzip " , no_argument , 0 , ' z ' } ,
{ " gzip-all " , no_argument , 0 , ' G ' } , { " gzip-all " , no_argument , 0 , ' G ' } ,
{ " level " , required_argument , 0 , ' l ' } , { " level " , required_argument , 0 , ' l ' } ,
{ " gzip-exts " , required_argument , 0 , ' g ' } , { " gzip-exts " , required_argument , 0 , ' g ' } ,
{ " input " , required_argument , 0 , ' i ' } , { " input " , required_argument , 0 , ' i ' } ,
{ " output " , required_argument , 0 , ' o ' } , { " output " , required_argument , 0 , ' o ' } ,
{ " help " , no_argument , 0 , ' h ' } , { " strip-path " , required_argument , 0 , ' S ' } ,
{ 0 , 0 , 0 , 0 } { " help " , no_argument , 0 , ' h ' } ,
{ 0 , 0 , 0 , 0 }
} ; } ;
c = getopt_long ( argc , argv , " c:l:g:zGhp:i:o:0123456789 " , c = getopt_long ( argc , argv , " c:l:g:zGhp:i:o:0123456789 " ,
@ -423,6 +412,10 @@ int main(int argc, char **argv)
parseFile = strdup ( optarg ) ; parseFile = strdup ( optarg ) ;
break ; break ;
case ' S ' :
stripPath = strdup ( optarg ) ;
break ;
case ' c ' : case ' c ' :
compType = atoi ( optarg ) ; compType = atoi ( optarg ) ;
break ; break ;
@ -434,11 +427,7 @@ int main(int argc, char **argv)
case ' g ' : case ' g ' :
use_gzip = true ; use_gzip = true ;
if ( ! parseGzipExtensions ( optarg ) ) { parseGzipExtensions ( optarg ) ;
fprintf ( stderr , " Bad gzip extension list: %s \n " , optarg ) ;
err = 1 ;
goto show_help ;
}
break ; break ;
case ' l ' : case ' l ' :
@ -473,7 +462,6 @@ int main(int argc, char **argv)
exit ( 0 ) ; exit ( 0 ) ;
} }
if ( s_gzipExtensions = = NULL & & use_gzip ) { if ( s_gzipExtensions = = NULL & & use_gzip ) {
parseGzipExtensions ( strdup ( DEFAULT_GZIP_EXTS ) ) ; parseGzipExtensions ( strdup ( DEFAULT_GZIP_EXTS ) ) ;
} }
@ -514,14 +502,24 @@ int main(int argc, char **argv)
serr = stat ( name , & statBuf ) ; serr = stat ( name , & statBuf ) ;
if ( ( serr = = 0 ) & & S_ISREG ( statBuf . st_mode ) ) { if ( ( serr = = 0 ) & & S_ISREG ( statBuf . st_mode ) ) {
//Strip off './' or '/' madness.
//Strip off './' or '/' madness.
realName = name ; char * embeddedName = name ;
if ( name [ 0 ] = = ' . ' & & name [ 1 ] = = ' / ' ) { realName + = 2 ; }
if ( realName [ 0 ] = = ' / ' ) { realName + + ; }
f = open ( name , O_RDONLY ) ; f = open ( name , O_RDONLY ) ;
// relative path starting with ./, remove that
if ( embeddedName [ 0 ] = = ' . ' & & embeddedName [ 1 ] = = ' / ' ) {
embeddedName + = 2 ;
}
// remove prefix
if ( stripPath & & 0 = = strncmp ( embeddedName , stripPath , strlen ( stripPath ) ) ) {
embeddedName + = strlen ( stripPath ) ;
}
// remove leading slash, if any
if ( embeddedName [ 0 ] = = ' / ' ) { embeddedName + + ; }
if ( f > 0 ) { if ( f > 0 ) {
const char * compName = " unknown " ; const char * compName = " unknown " ;
rate = handleFile ( f , realName , compType , compLvl , & compName ) ; rate = handleFile ( f , embedded Name, compType , compLvl , & compName ) ;
fprintf ( stderr , " %s (%d%%, %s) \n " , realName , rate , compName ) ; fprintf ( stderr , " %s (%d%%, %s) \n " , embedded Name, rate , compName ) ;
close ( f ) ; close ( f ) ;
} else { } else {
perror ( name ) ; perror ( name ) ;
@ -550,7 +548,7 @@ int main(int argc, char **argv)
fprintf ( stderr , " [-c|--compress COMPRESSOR] \n 0 - None, 1 - Heatshrink (default) \n " ) ; fprintf ( stderr , " [-c|--compress COMPRESSOR] \n 0 - None, 1 - Heatshrink (default) \n " ) ;
fprintf ( stderr , " [-l|--level LEVEL] or [-0 through -9] \n compression level 1-9, higher is better but uses more RAM \n " ) ; fprintf ( stderr , " [-l|--level LEVEL] or [-0 through -9] \n compression level 1-9, higher is better but uses more RAM \n " ) ;
fprintf ( stderr , " [-z|--gzip] \n use gzip for files with extensions matching " DEFAULT_GZIP_EXTS " \n " ) ; fprintf ( stderr , " [-z|--gzip] \n use gzip for files with extensions matching " DEFAULT_GZIP_EXTS " \n " ) ;
fprintf ( stderr , " [-Z |--gzip-all] \n use gzip for all files \n " ) ; fprintf ( stderr , " [-G |--gzip-all] \n use gzip for all files \n " ) ;
fprintf ( stderr , " [-g|--gzip-exts GZIPPED_EXTENSIONS] \n use gzip for files with custom extensions, comma-separated \n " ) ; fprintf ( stderr , " [-g|--gzip-exts GZIPPED_EXTENSIONS] \n use gzip for files with custom extensions, comma-separated \n " ) ;
fprintf ( stderr , " [-i|--input FILE] \n Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin if not specified by args \n " ) ; fprintf ( stderr , " [-i|--input FILE] \n Input file, can be multiple. Files can also be passed at the end without -i, or as lines on stdin if not specified by args \n " ) ;
fprintf ( stderr , " [-o|--output FILE] \n Output file name; if not specified, outputs to stdout \n " ) ; fprintf ( stderr , " [-o|--output FILE] \n Output file name; if not specified, outputs to stdout \n " ) ;
