From 2bba5c13ede6444c84630f12cb9568d6db9bec29 Mon Sep 17 00:00:00 2001 From: Jeroen Domburg Date: Wed, 8 Oct 2014 13:40:08 +0200 Subject: [PATCH] Added embedded filesystem. Whee! --- Makefile | 9 +++ html/cat.jpeg | Bin 0 -> 3317 bytes html/test.html | 7 +++ html/test2.html | 7 +++ mkespfsimage/Makefile | 4 ++ mkespfsimage/espfsformat.h | 31 ++++++++++ mkespfsimage/main.c | 109 +++++++++++++++++++++++++++++++++++ user/espfs.c | 114 +++++++++++++++++++++++++++++++++++++ user/espfs.h | 14 +++++ user/httpd.c | 70 ++++++++++++++++++----- 10 files changed, 351 insertions(+), 14 deletions(-) create mode 100644 html/cat.jpeg create mode 100644 html/test.html create mode 100644 html/test2.html create mode 100644 mkespfsimage/Makefile create mode 100644 mkespfsimage/espfsformat.h create mode 100644 mkespfsimage/main.c create mode 100644 user/espfs.c create mode 100644 user/espfs.h diff --git a/Makefile b/Makefile index 7a9cd25..f3e8197 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,15 @@ flash: firmware/0x00000.bin firmware/0x40000.bin sleep 3 -$(ESPTOOL) --port $(ESPPORT) write_flash 0x40000 firmware/0x40000.bin +webpages.espfs: html/ mkespfsimage/mkespfsimage + cd html; find | ../mkespfsimage/mkespfsimage > ../webpages.espfs; cd .. + +mkespfsimage/mkespfsimage: mkespfsimage/ + make -C mkespfsimage + +htmlflash: webpages.espfs + -$(ESPTOOL) --port $(ESPPORT) write_flash 0x20000 webpages.espfs + clean: $(Q) rm -f $(APP_AR) $(Q) rm -f $(TARGET_OUT) diff --git a/html/cat.jpeg b/html/cat.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..c80740917ec4a844da3475826ec2877887591db4 GIT binary patch literal 3317 zcmb7>_dDB*`^Mi%h*qh@I(E!zDW`~Cl-j|^uD$o(dsL~tXHk2r=%G@pp;cS0(%9n& zF={kH9ix1EemH-?_w`)QPxo`**L_{j&HT+e0Mk^}Pz69B008|S;ARPk18zepZc#vQ z-=erhNqPGYH6sl*6%{orJp+u9lbxHJlO2KJy^nmzdryQPfe?}t5)qe_k&)pMR8Urs zdWw{mkpxjvQc_b&&sd<5XP05c!~0^$b1a1aCzy6Fel000aD{Ra3C zawvq10t|p|{Z^%502l;;fS?d46h!{-1qciQ$l&C(Tq009Zbdy?-PIA8)9K{QKjQ!DQ`^md*69r#2 zso3NQ^VR))CMqfsDt$gnjC*1J%0_6p`jBU3R#-=j#9(&%GuJM3guXXG_FLwfCe(LJ zD<7Mrtvda6br(nC+Z)N{pTcJmxBu2S)(k$h<|&#BN62+sim_%EpU*RaBd!Li1D28v z1N8G2GObOQO)NEBgXvTrYh$KAWL$1<>(-;q+rzxd8jqZ^Y(KOs+2_KOU{R{_&NKLA zB^L6jln=(n7J2;Tlh$Z2X2Z-*#o9Y*vLg&~iodA+qF=e>vN(Qu##-XTz}~~-x~sL4 zyWd#+Nxrf>`E`?)6h|@kSN$Zf9XkGzpWQ(7%SH<`@ns1yB&!BZszX@n>A8{f$Jh)# zvtE~GGSx?SRJn1wDxtN!)%cv8eeskWBY;K&<<)yQ!U<1!m@Jn!Dq=P zg~eQ1Nmh>sZw<=rVQ@9!)qB|{C{M^AX;n^7F?pdBu8#vaA~nvZnNqAKy{W5&=hsuZ z0`MWm!&0QZXpGji!t`2}&2EZtP0zi{Sa`b%e?ox#Jw4)ZgU4q4N~^`bS30_bMLL?z zvT#aBTu581*U1#D!pQJ?be=rviYMp$fuc89vCcbEZkote!^v$8fu!OyMn7+%Jh46G z&*&I0O&d_+O^FWt)bB>Kn=_KsFf4O$96mV0O4Lt!eS0Ft0=|lPX&x_58Qn@I6vY)^$_YB7UsIpl%R{p>zv`fX@E{}CqMhVr z+-?A;YSTbvjPasm3)A=K4<3Lu`(&2)+J#@}zdC6?De-;Sry)L?)+Cmbnru7v_)4 zqSKJO`E(YkXOwE&t3mkXu&W2Ao6DP1W| z+3M>(z{e9Kzz1O3JSPRXAC4K6$I43EYleYP@?;~8A(8$vT+B1h2;ki&P;v~h7 zt2cMW@ZemTRBsHUn27~zP+GF*=q6o;+cO^xX$u1dH@@s}QtLa}oHj4c18Xi)O%i=o zYZ;GK`k(y;{5spYqb8Zq?Um$HCHL8Br<%s)J622CIi*?095Sr13Snu^@6xVGrp?7cP__Z43q3LA7W2em zXVVvN^+B5vx87u$$*NlWcolz;^}Qx^v|`)UyF0Rf6&C-9EN3`3nh_h2LRGsgJnxhG zel$cD{j)%jnMK}2&DB{D*~C6yrV!OjlQ8>jyVqq}>xj)&|Fi(~H?wjL_Zc>2fRg%EguINoHl;CqxIF=i=QNOLBXU`mFYi;LYAcH%s$& zRa9b zQdi^=bzDz*{ge)vXr(e?(vKqqON>pjbq>OunRe8$2A4)X__HSnUG2bnOLKCoBU*K} zKQl_SD)%eeU>Vw?LvSa%;?~bOWzq412m51GHC*cWRZ)~psxx~O=9vbzGdEcK9@k?A z2pZM2GT$%p(Bt@cvYl`5P0L7y%RESs4O2T=x9F1jj$YIGCTn;C-<9UPc%?XC|o z?gbi1S7*^?X3^XDNe&ZyjnIy3UCnehX_v*wLqP1gSgL<%uT0c*py$1jYz_YQw7Z6h zTddglup6Liw04BPtJvP=s0eml ze94^#v!>X%o8L~(=Fc)1Om{7}h5#(~( za?tTn<0ub6Ll*~;Gk;40V!2d%+hZ}Ogq$^#te!ols*=nDCzo0L+}N+Ps>m{2br04= zoFN|ehubiSVrl+r&ja^!2=8zfWagOLI?4V)55-R{m>P3%G*pm(pM7PRED>enJjf%| z{`11%ZiTaC#0}th=oS9{@baS{x74sy%_Ww_H2o5*ooAVNzjH#ZCxq0bT4q{z@^WmK zkT>C&H`KuDPa3#g45UUrh}E4zADCwGg@SJY^BmN1Ys-&|QwA>?G@V1Qqiin$vM2cM zd&NK6VWn}ez47P>-i@{jXSZ`9&pT{^BFTg!tr^@Gh87IPtWxhrLZ;SFTu@y0UBM(i zS6<=%Y?qo75#AlOBKb1&KO>E>=#d*Bt#Q0B#vTqh zWR=zUtKWZf2gIGv46cn2O5V{#w{6!m)mK97Ux!`i9e+)Pv;@@|q$2Gif8<{Y&2bdI znf}@KF2|R?*qnnQ@*|@3fqqYV?1rm1J)$DnBd8VkuueOlbmkF__kr#oTWa`(j}I$J zDqWWesOvv +Test + +

Test!

+

Yay, the cpio filesystem thingy works. Click here to see +if the 2nd page works too. +

diff --git a/html/test2.html b/html/test2.html new file mode 100644 index 0000000..990b231 --- /dev/null +++ b/html/test2.html @@ -0,0 +1,7 @@ + +Test + +

Test2!

+

Here's an image of a cat (hopefully...)
+ +

diff --git a/mkespfsimage/Makefile b/mkespfsimage/Makefile new file mode 100644 index 0000000..07b61a8 --- /dev/null +++ b/mkespfsimage/Makefile @@ -0,0 +1,4 @@ + + +mkespfsimage: main.o + $(CC) -o mkespfsimage $< diff --git a/mkespfsimage/espfsformat.h b/mkespfsimage/espfsformat.h new file mode 100644 index 0000000..10a7d2b --- /dev/null +++ b/mkespfsimage/espfsformat.h @@ -0,0 +1,31 @@ +#ifndef ESPROFSFORMAT_H +#define ESPROFSFORMAT_H + +/* +Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module. +Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files, +headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself +when trying to do a <4byte or unaligned read. +*/ + +/* +The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data. +Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header +with the FLAG_LASTFILE flag set. +*/ + + +#define FLAG_LASTFILE (1<<0) +#define COMPRESS_NONE 0 +#define COMPRESS_LZF 1 + +typedef struct { + int32_t magic; + int8_t flags; + int8_t compression; + int16_t nameLen; + int32_t fileLenComp; + int32_t fileLenDecomp; +} EspFsHeader; + +#endif \ No newline at end of file diff --git a/mkespfsimage/main.c b/mkespfsimage/main.c new file mode 100644 index 0000000..d6a6caf --- /dev/null +++ b/mkespfsimage/main.c @@ -0,0 +1,109 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "espfsformat.h" + +//Routines to convert host format to the endianness used in the xtensa +short htoxs(short in) { + char r[2]; + r[0]=in; + r[1]=in>>8; + return *((short *)r); +} + +int htoxl(int in) { + unsigned char r[4]; + r[0]=in; + r[1]=in>>8; + r[2]=in>>16; + r[3]=in>>24; + return *((int *)r); +} + + +void handleFile(int f, char *name) { + char *fdat; + off_t size; + EspFsHeader h; + int nameLen; + size=lseek(f, 0, SEEK_END); + fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0); + if (fdat==MAP_FAILED) { + perror("mmap"); + return; + } + + //Fill header data + h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24); + h.flags=0; + h.compression=COMPRESS_NONE; + nameLen=strlen(name)+1; + if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary + h.nameLen=htoxs(nameLen); + h.fileLenComp=htoxl(size); + h.fileLenDecomp=htoxl(size); + + write(1, &h, sizeof(EspFsHeader)); + write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer. + write(1, fdat, size); + //Pad out to 32bit boundary + while (size&3) { + write(1, "\000", 1); + size++; + } + munmap(fdat, size); +} + +//Write final dummy header with FLAG_LASTFILE set. +void finishArchive() { + EspFsHeader h; + h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24); + h.flags=FLAG_LASTFILE; + h.compression=COMPRESS_NONE; + h.nameLen=htoxs(0); + h.fileLenComp=htoxl(0); + h.fileLenDecomp=htoxl(0); + write(1, &h, sizeof(EspFsHeader)); +} + + +int main(int argc, char **argv) { + int f; + char fileName[1024]; + char *realName; + struct stat statBuf; + int serr; + while(fgets(fileName, sizeof(fileName), stdin)) { + //Kill off '\n' at the end + fileName[strlen(fileName)-1]=0; + //Only include files + serr=stat(fileName, &statBuf); + if ((serr==0) && S_ISREG(statBuf.st_mode)) { + //Strip off './' or '/' madness. + realName=fileName; + if (fileName[0]=='.') realName++; + if (realName[0]=='/') realName++; + f=open(fileName, O_RDONLY); + if (f>0) { + handleFile(f, realName); + fprintf(stderr, "%s\n", realName); + close(f); + } else { + perror(fileName); + } + } else { + if (serr!=0) { + perror(fileName); + } + } + } + finishArchive(); +} + diff --git a/user/espfs.c b/user/espfs.c new file mode 100644 index 0000000..8a6ffc6 --- /dev/null +++ b/user/espfs.c @@ -0,0 +1,114 @@ +#include "driver/uart.h" +#include "c_types.h" +#include "user_interface.h" +#include "espconn.h" +#include "mem.h" +#include "osapi.h" +#include "../mkespfsimage/espfsformat.h" +#include "espfs.h" + +struct EspFsFile { + EspFsHeader *header; + char decompressor; + int32_t posDecomp; + char *posStart; + char *posComp; + void *decompData; +}; + +/* +Available locations, at least in my flash, with boundaries partially guessed: +0x00000 (0x10000): Code/data (RAM data?) +0x10000 (0x30000): Free (filled with zeroes) (parts used by ESPCloud and maybe SSL) +0x40000 (0x20000): Code/data (ROM data?) +0x60000 (0x1C000): Free +0x7c000 (0x04000): Param store +0x80000 - end of flash + +Accessing the flash through the mem emulation at 0x40200000 is a bit hairy: All accesses +*must* be aligned 32-bit accesses. Reading a short, byte or unaligned word will result in +a memory exception, crashing the program. +*/ + + +//Copies len bytes over from dst to src, but does it using *only* +//aligned 32-bit reads. +void memcpyAligned(char *dst, char *src, int len) { + int x; + int w, b; + for (x=0; x>0); + if (b==1) *dst=(w>>8); + if (b==2) *dst=(w>>16); + if (b==3) *dst=(w>>24); + dst++; src++; + } +} + + + +EspFsFile *espFsOpen(char *fileName) { + char *p=(char *)(ESPFS_POS+0x40200000); + char *hpos; + char namebuf[256]; + EspFsHeader h; + EspFsFile *r; + //Skip initial slashes + while(fileName[0]=='/') fileName++; + //Go find that file! + while(1) { + hpos=p; + os_memcpy(&h, p, sizeof(EspFsHeader)); + //ToDo: check magic + if (h.flags&FLAG_LASTFILE) { +// os_printf("End of archive.\n"); + return NULL; + } + p+=sizeof(EspFsHeader); + os_memcpy(namebuf, p, sizeof(namebuf)); +// os_printf("Found file %s . Namelen=%x fileLen=%x\n", namebuf, h.nameLen,h.fileLenComp); + if (os_strcmp(namebuf, fileName)==0) { + p+=h.nameLen; + r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); + if (r==NULL) return NULL; + r->header=(EspFsHeader *)hpos; + r->decompressor=h.compression; + r->posComp=p; + r->posStart=p; + r->posDecomp=0; + r->decompData=NULL; + //ToDo: Init any decompressor + return r; + } + //Skip name and file + p+=h.nameLen+h.fileLenComp; + if ((int)p&3) p+=4-((int)p&3); //align to next 32bit val +// os_printf("Next addr = %x\n", (int)p); + } +} + + +int espFsRead(EspFsFile *fh, char *buff, int len) { + if (fh==NULL) return 0; + if (fh->decompressor==COMPRESS_NONE) { + int toRead; + toRead=fh->header->fileLenComp-(fh->posComp-fh->posStart); + if (len>toRead) len=toRead; +// os_printf("Reading %d bytes from %x\n", len, fh->posComp); + memcpyAligned(buff, fh->posComp, len); + fh->posDecomp+=len; + fh->posComp+=len; +// os_printf("Done reading %d bytes, pos=%x\n", len, fh->posComp); + return len; + } +} + +void espFsClose(EspFsFile *fh) { + if (fh==NULL) return; + os_free(fh); +} + + + diff --git a/user/espfs.h b/user/espfs.h new file mode 100644 index 0000000..e479f90 --- /dev/null +++ b/user/espfs.h @@ -0,0 +1,14 @@ +#ifndef ESPFS_H +#define ESPFS_H + +//Pos of esp fs in flash +#define ESPFS_POS 0x20000 + +typedef struct EspFsFile EspFsFile; + +EspFsFile *espFsOpen(char *fileName); +int espFsRead(EspFsFile *fh, char *buff, int len); +void espFsClose(EspFsFile *fh); + + +#endif \ No newline at end of file diff --git a/user/httpd.c b/user/httpd.c index 59417b6..ccf695a 100644 --- a/user/httpd.c +++ b/user/httpd.c @@ -8,18 +8,34 @@ #include "espconn.h" #include "httpd.h" #include "io.h" +#include "espfs.h" #define MAX_HEAD_LEN 1024 -int cgiSet(struct espconn *conn); - -typedef int (* cgiSendCallback)(struct espconn *conn); +//struct UrlData; +typedef struct UrlData UrlData; typedef struct { + char *url; + char *getArgs; + const UrlData *effUrl; + char *datPtr; + int datLen; + EspFsFile *file; +} GetData; + +typedef int (* cgiSendCallback)(struct espconn *conn, GetData *getData); + +struct UrlData { const char *url; const char *fixedResp; cgiSendCallback cgiCb; -} UrlData; +}; + +int cgiSet(struct espconn *conn, GetData *getData); +int cgiGetFlash(struct espconn *conn, GetData *getData); +static int cgiSendFile(struct espconn *conn, GetData *getData); + const char htmlIndex[]="Hello World \

Hello, World!

\ @@ -29,21 +45,18 @@ const char htmlIndex[]="Hello World \ static const UrlData urls[]={ {"/", htmlIndex, NULL}, {"/set", NULL, cgiSet}, + {"/flash.bin", NULL, cgiGetFlash}, {NULL, NULL, NULL}, }; +static const UrlData cpioUrlData={"*", NULL, cgiSendFile}; + static struct espconn conn; static esp_tcp tcp; static int recLen; static char recBuff[MAX_HEAD_LEN]; -typedef struct { - char *url; - char *getArgs; - const UrlData *effUrl; -} GetData; - static GetData getData; //Find a specific arg in a string of get- or post-data. @@ -70,18 +83,35 @@ static char *ICACHE_FLASH_ATTR httpdFindArg(char *line, char *arg) { } //ToDo: Move cgi functions to somewhere else -int cgiSet(struct espconn *conn) { +int ICACHE_FLASH_ATTR cgiSet(struct espconn *conn, GetData *getData) { char *on; static const char okStr[]="OK

OK

"; - on=httpdFindArg(getData.getArgs, "led"); + on=httpdFindArg(getData->getArgs, "led"); os_printf("cgiSet: on=%s\n", on?on:"not found"); if (on!=NULL) ioLed(atoi(on)); espconn_sent(conn, (uint8 *)okStr, os_strlen(okStr)); return 1; } +int ICACHE_FLASH_ATTR cgiGetFlash(struct espconn *conn, GetData *getData) { + static char *p=(char *)0x40200000; + static int t=0; + espconn_sent(conn, (uint8 *)p, 1024); + p+=1024; + t++; + if (t<1024) return 0; + t=0; + p=(char*)0x40200000; + return 1; +} - +static int ICACHE_FLASH_ATTR cgiSendFile(struct espconn *conn, GetData *getData) { + int len; + char buff[1024]; + len=espFsRead(getData->file, buff, 1024); + espconn_sent(conn, (uint8 *)buff, len); + return (len==0); +} static const char *httpOkHeader="HTTP/1.0 200 OK\r\nServer: esp8266-thingie/0.1\r\nContent-Type: text/html\r\n\r\n"; static const char *httpNotFoundHeader="HTTP/1.0 404 Not Found\r\nServer: esp8266-thingie/0.1\r\n\r\nNot Found.\r\n"; @@ -94,7 +124,7 @@ static void ICACHE_FLASH_ATTR httpdSentCb(void *arg) { } if (getData.effUrl->cgiCb!=NULL) { - if (getData.effUrl->cgiCb(conn)) getData.effUrl=NULL; + if (getData.effUrl->cgiCb(conn, &getData)) getData.effUrl=NULL; } else { espconn_sent(conn, (uint8 *)getData.effUrl->fixedResp, os_strlen(getData.effUrl->fixedResp)); getData.effUrl=NULL; @@ -103,6 +133,8 @@ static void ICACHE_FLASH_ATTR httpdSentCb(void *arg) { static void ICACHE_FLASH_ATTR httpdSendResp(struct espconn *conn) { int i=0; + EspFsFile *fdat; + //See if the url is somewhere in our internal url table. while (urls[i].url!=NULL) { if (os_strcmp(urls[i].url, getData.url)==0) { getData.effUrl=&urls[i]; @@ -112,6 +144,16 @@ static void ICACHE_FLASH_ATTR httpdSendResp(struct espconn *conn) { } i++; } + //Nope. See if it's in the cpio archive + fdat=espFsOpen(getData.url); + if (fdat!=NULL) { + //Found + getData.file=fdat; + getData.effUrl=&cpioUrlData; + espconn_sent(conn, (uint8 *)httpOkHeader, os_strlen(httpOkHeader)); + return; + } + //Can't find :/ espconn_sent(conn, (uint8 *)httpNotFoundHeader, os_strlen(httpNotFoundHeader)); }