Moved espfs files and tools together, fixed espfstest

This commit is contained in:
Jindra Dolezy
2015-04-11 00:35:54 +02:00
parent 99e72f5915
commit c3f3cbcf9c
21 changed files with 57 additions and 62 deletions
+238
View File
@@ -0,0 +1,238 @@
/*
This is a simple read-only implementation of a file system. It uses a block of data coming from the
mkespfsimg tool, and can use that block to do abstracted operations on the files that are in there.
It's written for use with httpd, but doesn't need to be used as such.
*/
/*
* ----------------------------------------------------------------------------
* "THE BEER-WARE LICENSE" (Revision 42):
* Jeroen Domburg <jeroen@spritesmods.com> wrote this file. As long as you retain
* this notice you can do whatever you want with this stuff. If we meet some day,
* and you think this stuff is worth it, you can buy me a beer in return.
* ----------------------------------------------------------------------------
*/
//These routines can also be tested by comping them in with the espfstest tool. This
//simplifies debugging, but needs some slightly different headers. The #ifdef takes
//care of that.
#ifdef __ets__
//esp build
#include "c_types.h"
#include "user_interface.h"
#include "espconn.h"
#include "mem.h"
#include "osapi.h"
#include "espmissingincludes.h"
#else
//Test build
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#define os_malloc malloc
#define os_free free
#define os_memcpy memcpy
#define os_strncmp strncmp
#define os_strcmp strcmp
#define os_strcpy strcpy
#define os_printf printf
#define ICACHE_FLASH_ATTR
extern char* espFsData;
#endif
#include "espfsformat.h"
#include "espfs.h"
#ifdef ESPFS_HEATSHRINK
#include "heatshrink_config_custom.h"
#include "heatshrink_decoder.h"
#endif
struct EspFsFile {
EspFsHeader *header;
char decompressor;
int32_t posDecomp;
char *posStart;
char *posComp;
void *decompData;
};
/*
Available locations, at least in my flash, with boundaries partially guessed. This
is using 0.9.1/0.9.2 SDK on a not-too-new module.
0x00000 (0x10000): Code/data (RAM data?)
0x10000 (0x02000): Gets erased by something?
0x12000 (0x2E000): Free (filled with zeroes) (parts used by ESPCloud and maybe SSL)
0x40000 (0x20000): Code/data (ROM data?)
0x60000 (0x1C000): Free
0x7c000 (0x04000): Param store
0x80000 - end of flash
Accessing the flash through the mem emulation at 0x40200000 is a bit hairy: All accesses
*must* be aligned 32-bit accesses. Reading a short, byte or unaligned word will result in
a memory exception, crashing the program.
*/
//Copies len bytes over from dst to src, but does it using *only*
//aligned 32-bit reads. Yes, it's no too optimized but it's short and sweet and it works.
//ToDo: perhaps os_memcpy also does unaligned accesses?
#ifdef __ets__
void ICACHE_FLASH_ATTR memcpyAligned(char *dst, char *src, int len) {
int x;
int w, b;
for (x=0; x<len; x++) {
b=((int)src&3);
w=*((int *)(src-b));
if (b==0) *dst=(w>>0);
if (b==1) *dst=(w>>8);
if (b==2) *dst=(w>>16);
if (b==3) *dst=(w>>24);
dst++; src++;
}
}
#else
#define memcpyAligned memcpy
#endif
//Open a file and return a pointer to the file desc struct.
EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
#ifdef __ets__
char *p=(char *)(ESPFS_POS+0x40200000);
#else
char *p=espFsData;
#endif
char *hpos;
char namebuf[256];
EspFsHeader h;
EspFsFile *r;
//Strip initial slashes
while(fileName[0]=='/') fileName++;
//Go find that file!
while(1) {
hpos=p;
//Grab the next file header.
os_memcpy(&h, p, sizeof(EspFsHeader));
if (h.magic!=0x73665345) {
os_printf("Magic mismatch. EspFS image broken.\n");
return NULL;
}
if (h.flags&FLAG_LASTFILE) {
os_printf("End of image.\n");
return NULL;
}
//Grab the name of the file.
p+=sizeof(EspFsHeader);
os_memcpy(namebuf, p, sizeof(namebuf));
// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n",
// namebuf, (unsigned int)h.nameLen, (unsigned int)h.fileLenComp, h.compression, h.flags);
if (os_strcmp(namebuf, fileName)==0) {
//Yay, this is the file we need!
p+=h.nameLen; //Skip to content.
r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem
// os_printf("Alloc %p\n", r);
if (r==NULL) return NULL;
r->header=(EspFsHeader *)hpos;
r->decompressor=h.compression;
r->posComp=p;
r->posStart=p;
r->posDecomp=0;
if (h.compression==COMPRESS_NONE) {
r->decompData=NULL;
#ifdef ESPFS_HEATSHRINK
} else if (h.compression==COMPRESS_HEATSHRINK) {
//File is compressed with Heatshrink.
char parm;
heatshrink_decoder *dec;
//Decoder params are stored in 1st byte.
memcpyAligned(&parm, r->posComp, 1);
r->posComp++;
os_printf("Heatshrink compressed file; decode parms = %x\n", parm);
dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf);
r->decompData=dec;
#endif
} else {
os_printf("Invalid compression: %d\n", h.compression);
return NULL;
}
return r;
}
//We don't need this file. Skip name and file
p+=h.nameLen+h.fileLenComp;
if ((int)p&3) p+=4-((int)p&3); //align to next 32bit val
}
}
//Read len bytes from the given file into buff. Returns the actual amount of bytes read.
int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) {
int flen;
if (fh==NULL) return 0;
//Cache file length.
memcpyAligned((char*)&flen, (char*)&fh->header->fileLenComp, 4);
//Do stuff depending on the way the file is compressed.
if (fh->decompressor==COMPRESS_NONE) {
int toRead;
toRead=flen-(fh->posComp-fh->posStart);
if (len>toRead) len=toRead;
// os_printf("Reading %d bytes from %x\n", len, (unsigned int)fh->posComp);
memcpyAligned(buff, fh->posComp, len);
fh->posDecomp+=len;
fh->posComp+=len;
// os_printf("Done reading %d bytes, pos=%x\n", len, fh->posComp);
return len;
#ifdef ESPFS_HEATSHRINK
} else if (fh->decompressor==COMPRESS_HEATSHRINK) {
int decoded=0;
size_t elen, rlen;
char ebuff[16];
heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData;
// os_printf("Alloc %p\n", dec);
while(decoded<len) {
//Feed data into the decompressor
//ToDo: Check ret val of heatshrink fns for errors
elen=flen-(fh->posComp - fh->posStart);
if (elen==0) return decoded; //file is read
if (elen>0) {
memcpyAligned(ebuff, fh->posComp, 16);
heatshrink_decoder_sink(dec, (uint8_t *)ebuff, (elen>16)?16:elen, &rlen);
fh->posComp+=rlen;
if (rlen==elen) {
heatshrink_decoder_finish(dec);
}
}
//Grab decompressed data and put into buff
heatshrink_decoder_poll(dec, (uint8_t *)buff, len-decoded, &rlen);
fh->posDecomp+=rlen;
buff+=rlen;
decoded+=rlen;
}
return len;
#endif
}
return 0;
}
//Close the file.
void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) {
if (fh==NULL) return;
#ifdef ESPFS_HEATSHRINK
if (fh->decompressor==COMPRESS_HEATSHRINK) {
heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData;
heatshrink_decoder_free(dec);
// os_printf("Freed %p\n", dec);
}
#endif
// os_printf("Freed %p\n", fh);
os_free(fh);
}
+19
View File
@@ -0,0 +1,19 @@
#ifndef ESPFS_H
#define ESPFS_H
//Define this if you want to be able to use Heatshrink-compressed espfs images.
#define ESPFS_HEATSHRINK
//Pos of esp fs in flash
#define ESPFS_POS 0x12000
#define ESPFS_SIZE 0x2E000
typedef struct EspFsFile EspFsFile;
EspFsFile *espFsOpen(char *fileName);
int espFsRead(EspFsFile *fh, char *buff, int len);
void espFsClose(EspFsFile *fh);
#endif
+31
View File
@@ -0,0 +1,31 @@
#ifndef ESPROFSFORMAT_H
#define ESPROFSFORMAT_H
/*
Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module.
Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files,
headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself
when trying to do a <4byte or unaligned read.
*/
/*
The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data.
Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header
with the FLAG_LASTFILE flag set.
*/
#define FLAG_LASTFILE (1<<0)
#define COMPRESS_NONE 0
#define COMPRESS_HEATSHRINK 1
typedef struct {
int32_t magic;
int8_t flags;
int8_t compression;
int16_t nameLen;
int32_t fileLenComp;
int32_t fileLenDecomp;
} __attribute__((packed)) EspFsHeader;
#endif
+13
View File
@@ -0,0 +1,13 @@
CFLAGS=-I../../lib/heatshrink -I.. -std=gnu99
espfstest: main.o espfs.o heatshrink_decoder.o
$(CC) -o $@ $^
espfs.o: ../espfs.c
$(CC) $(CFLAGS) -c $^ -o $@
heatshrink_decoder.o: ../heatshrink_decoder.c
$(CC) $(CFLAGS) -c $^ -o $@
clean:
rm -f *.o espfstest
+60
View File
@@ -0,0 +1,60 @@
/*
Simple and stupid file decompressor for an espfs image. Mostly used as a testbed for espfs.c and
the decompressors: code compiled natively is way easier to debug using gdb et all :)
*/
#include <stdio.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include "espfs.h"
char *espFsData;
int main(int argc, char **argv) {
int f, out;
int len;
char buff[128];
EspFsFile *ef;
off_t size;
if (argc!=3) {
printf("Usage: %s espfs-image file\nExpands file from the espfs-image archive.\n", argv[0]);
exit(0);
}
f=open(argv[1], O_RDONLY);
if (f<=0) {
perror(argv[1]);
exit(1);
}
size=lseek(f, 0, SEEK_END);
espFsData=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0);
if (espFsData==MAP_FAILED) {
perror("mmap");
exit(1);
}
ef=espFsOpen(argv[2]);
if (ef==NULL) {
printf("Couldn't find %s in image.\n", argv[2]);
exit(1);
}
out=open(argv[2], O_WRONLY|O_CREAT|O_TRUNC, 0644);
if (out<=0) {
perror(argv[2]);
exit(1);
}
while ((len=espFsRead(ef, buff, 128))!=0) {
write(out, buff, len);
}
espFsClose(ef);
//munmap, close, ... I can't be bothered.
}
+30
View File
@@ -0,0 +1,30 @@
//Heatshrink config for the decompressor.
#ifndef HEATSHRINK_CONFIG_H
#define HEATSHRINK_CONFIG_H
/* Should functionality assuming dynamic allocation be used? */
#define HEATSHRINK_DYNAMIC_ALLOC 1
#if HEATSHRINK_DYNAMIC_ALLOC
/* Optional replacement of malloc/free */
#ifdef __ets__
#define HEATSHRINK_MALLOC(SZ) os_malloc(SZ)
#define HEATSHRINK_FREE(P, SZ) os_free(P)
#else
#define HEATSHRINK_MALLOC(SZ) malloc(SZ)
#define HEATSHRINK_FREE(P, SZ) free(P)
#endif
#else
/* Required parameters for static configuration */
#define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32
#define HEATSHRINK_STATIC_WINDOW_BITS 8
#define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4
#endif
/* Turn on logging for debugging. */
#define HEATSHRINK_DEBUGGING_LOGS 0
/* Use indexing for faster compression. (This requires additional space.) */
#define HEATSHRINK_USE_INDEX 1
#endif
+25
View File
@@ -0,0 +1,25 @@
#include "espfs.h"
#ifdef ESPFS_HEATSHRINK
//Stupid wrapper so we don't have to move c-files around
//Also loads httpd-specific config.
#ifdef __ets__
//esp build
#define _STDLIB_H_
#define _STRING_H_
#define _STDDEF_H
#include "espmissingincludes.h"
#include "c_types.h"
#include "mem.h"
#include "osapi.h"
#define memset(x,y,z) os_memset(x,y,z)
#define memcpy(x,y,z) os_memcpy(x,y,z)
#endif
#include "heatshrink_config_custom.h"
#include "../lib/heatshrink/heatshrink_decoder.c"
#endif
+10
View File
@@ -0,0 +1,10 @@
CFLAGS=-I../../lib/heatshrink -I.. -std=gnu99
OBJS=main.o heatshrink_encoder.o
TARGET=mkespfsimage
$(TARGET): $(OBJS)
$(CC) -o $@ $^
clean:
rm -f $(TARGET) $(OBJS)
+3
View File
@@ -0,0 +1,3 @@
//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir
#include "../lib/heatshrink/heatshrink_encoder.c"
+205
View File
@@ -0,0 +1,205 @@
#include <stdint.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <arpa/inet.h>
#include <string.h>
#include "espfsformat.h"
//Heatshrink
#include "heatshrink_common.h"
#include "heatshrink_config.h"
#include "heatshrink_encoder.h"
//Routines to convert host format to the endianness used in the xtensa
short htoxs(short in) {
char r[2];
r[0]=in;
r[1]=in>>8;
return *((short *)r);
}
int htoxl(int in) {
unsigned char r[4];
r[0]=in;
r[1]=in>>8;
r[2]=in>>16;
r[3]=in>>24;
return *((int *)r);
}
size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) {
char *inp=in;
char *outp=out;
size_t len;
int ws[]={5, 6, 8, 11, 13};
int ls[]={3, 3, 4, 4, 4};
HSE_poll_res pres;
HSE_sink_res sres;
size_t r;
if (level==-1) level=8;
level=(level-1)/2; //level is now 0, 1, 2, 3, 4
heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]);
if (enc==NULL) {
perror("allocating mem for heatshrink");
exit(1);
}
//Save encoder parms as first byte
*outp=(ws[level]<<4)|ls[level];
outp++; outsize--;
r=1;
do {
if (insize>0) {
sres=heatshrink_encoder_sink(enc, inp, insize, &len);
if (sres!=HSER_SINK_OK) break;
inp+=len; insize-=len;
if (insize==0) heatshrink_encoder_finish(enc);
}
do {
pres=heatshrink_encoder_poll(enc, outp, outsize, &len);
if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break;
outp+=len; outsize-=len;
r+=len;
} while (pres==HSER_POLL_MORE);
} while (insize!=0);
if (insize!=0) {
fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres);
exit(1);
}
heatshrink_encoder_free(enc);
return r;
}
int handleFile(int f, char *name, int compression, int level) {
char *fdat, *cdat;
off_t size, csize;
EspFsHeader h;
int nameLen;
size=lseek(f, 0, SEEK_END);
fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0);
if (fdat==MAP_FAILED) {
perror("mmap");
return 0;
}
if (compression==COMPRESS_NONE) {
csize=size;
cdat=fdat;
} else if (compression==COMPRESS_HEATSHRINK) {
cdat=malloc(size*2);
csize=compressHeatshrink(fdat, size, cdat, size*2, level);
} else {
fprintf(stderr, "Unknown compression - %d\n", compression);
exit(1);
}
if (csize>size) {
//Compressing enbiggened this file. Revert to uncompressed store.
compression=COMPRESS_NONE;
csize=size;
cdat=fdat;
}
//Fill header data
h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24);
h.flags=0;
h.compression=compression;
nameLen=strlen(name)+1;
if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary
h.nameLen=htoxs(nameLen);
h.fileLenComp=htoxl(csize);
h.fileLenDecomp=htoxl(size);
write(1, &h, sizeof(EspFsHeader));
write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer.
write(1, cdat, csize);
//Pad out to 32bit boundary
while (csize&3) {
write(1, "\000", 1);
csize++;
}
munmap(fdat, size);
return (csize*100)/size;
}
//Write final dummy header with FLAG_LASTFILE set.
void finishArchive() {
EspFsHeader h;
h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24);
h.flags=FLAG_LASTFILE;
h.compression=COMPRESS_NONE;
h.nameLen=htoxs(0);
h.fileLenComp=htoxl(0);
h.fileLenDecomp=htoxl(0);
write(1, &h, sizeof(EspFsHeader));
}
int main(int argc, char **argv) {
int f, x;
char fileName[1024];
char *realName;
struct stat statBuf;
int serr;
int rate;
int err=0;
int compType=1; //default compression type - heatshrink
int compLvl=-1;
for (x=1; x<argc; x++) {
if (strcmp(argv[x], "-c")==0 && argc>=x-2) {
compType=atoi(argv[x=1]);
x++;
} else if (strcmp(argv[x], "-l")==0 && argc>=x-2) {
compLvl=atoi(argv[x=1]);
if (compLvl<1 || compLvl>9) err=1;
x++;
} else {
err=1;
}
}
if (err) {
fprintf(stderr, "%s - Program to create espfs images\n", argv[0]);
fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] > out.espfs\n", argv[0]);
fprintf(stderr, "Compressors:\n0 - None\n1 - Heatshrink(defautl\n");
fprintf(stderr, "Compression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n");
exit(0);
}
while(fgets(fileName, sizeof(fileName), stdin)) {
//Kill off '\n' at the end
fileName[strlen(fileName)-1]=0;
//Only include files
serr=stat(fileName, &statBuf);
if ((serr==0) && S_ISREG(statBuf.st_mode)) {
//Strip off './' or '/' madness.
realName=fileName;
if (fileName[0]=='.') realName++;
if (realName[0]=='/') realName++;
f=open(fileName, O_RDONLY);
if (f>0) {
rate=handleFile(f, realName, compType, compLvl);
fprintf(stderr, "%s (%d%%)\n", realName, rate);
close(f);
} else {
perror(fileName);
}
} else {
if (serr!=0) {
perror(fileName);
}
}
}
finishArchive();
return 0;
}