add webserver components, some webserver fices and improvements in templating, add real time history chart

components/fileserver/CMakeLists.txt

@@ -0,0 +1,9 @@
+set(COMPONENT_ADD_INCLUDEDIRS
+        "include")
+
+set(COMPONENT_SRCDIRS
+        "src")
+
+set(COMPONENT_REQUIRES tcpip_adapter esp_http_server httpd_utils common_utils)
+
+register_component()

components/fileserver/README.txt

@@ -0,0 +1,2 @@
+File and template serving support for the http_server bundled with ESP-IDF.
+

components/fileserver/component.mk

@@ -0,0 +1,3 @@
+
+COMPONENT_SRCDIRS := src
+COMPONENT_ADD_INCLUDEDIRS := include

components/fileserver/include/fileserver/embedded_files.h

@@ -0,0 +1,51 @@
+//
+// Created on 2018/10/17 by Ondrej Hruska <ondra@ondrovo.com>
+//
+
+#ifndef FBNODE_EMBEDDED_FILES_H
+#define FBNODE_EMBEDDED_FILES_H
+
+#include <stdint.h>
+#include <esp_err.h>
+#include <stdbool.h>
+
+struct embedded_file_info {
+    const uint8_t * start;
+    const uint8_t * end;
+    const char * name;
+    const char * mime;
+};
+
+enum file_access_level {
+    /** Public = file accessed by a wildcard route */
+    FILE_ACCESS_PUBLIC = 0,
+    /** Protected = file included in a template or explicitly specified in a route */
+    FILE_ACCESS_PROTECTED = 1,
+    /** Files protected against read-out */
+    FILE_ACCESS_PRIVATE = 2,
+};
+
+extern const struct embedded_file_info EMBEDDED_FILE_LOOKUP[];
+extern const size_t EMBEDDED_FILE_LOOKUP_LEN;
+
+/**
+ * Find an embedded file by its name.
+ * 
+ * This function is weak. It crawls the EMBEDDED_FILE_LOOKUP table and checks for exact match, also
+ * testing with www_get_static_file_access_check if the access is allowed.
+ *
+ * @param name - file name
+ * @param access - access level (public - wildcard fallthrough, protected - files for the server, loaded explicitly)
+ * @param[out] file - the file struct is stored here if found, unchanged if not found.
+ * @return status code
+ */
+esp_err_t www_get_static_file(const char *name, enum file_access_level access, const struct embedded_file_info **file);
+
+/**
+ * Check file access permission (if using the default www_get_static_file implementation).
+ * 
+ * This function is weak. The default implementation returns always true.
+ */
+bool www_get_static_file_access_check(const struct embedded_file_info *file, enum file_access_level access);
+
+#endif //FBNODE_EMBEDDED_FILES_H

components/fileserver/include/fileserver/token_subs.h

@@ -0,0 +1,227 @@
+//
+// This module implements token substitution in files served by the server.
+//
+// Tokens are in the form {token}, or {escape:token}, where escape can be:
+// - h ... html escape (plain text in a html file, attribute value)
+// - j ... js escape (for use in JS strings)
+//
+// When no escape is specified, the token substitution is written verbatim into the response.
+//
+//   var foo = "{j:foo}";
+//   <input value="{h:old_value}">
+//   {generated-html-goes-here}
+//
+// Token can be made optional by adding '?' at the end (this can't be used for includes).
+// Such token then simply becomes empty string when not substituted, as opposed to being included in the page verbatim.
+//
+//   <input value="{h:old_value?}">
+//
+// token names can contain alnum, dash, period and underscore, and are case sensitive.
+//
+//
+// It is further possible to include a static file with optional key-value replacements. These serve as defaults.
+//
+//   {@_subfile.html}
+//   {@_subfile.html|key=value lalala}
+//   {@_subfile.html|key=value lalala|other=value}
+//
+// File inclusion can be nested, and the files can use replacement tokens as specified by the include statement
+//
+// Created on 2019/01/24 by Ondrej Hruska <ondra@ondrovo.com>
+//
+
+#ifndef FBNODE_TOKEN_SUBS_H
+#define FBNODE_TOKEN_SUBS_H
+
+#include "embedded_files.h"
+#include <rom/queue.h>
+#include <esp_err.h>
+#include <esp_http_server.h>
+
+/** Max length of a token buffer (must suffice for all included filenames) */
+#define MAX_TOKEN_LEN 32
+
+/** Max length of a key-value substitution when using tpl_kv_replacer;
+ * This is also used internally for in-line replacements in file imports. */
+#define TPL_KV_KEY_LEN 24
+/** Max length of a substituion in tpl_kv_replacer */
+#define TPL_KV_SUBST_LEN 64
+
+/**
+ * Escape type - argument for httpd_resp_send_chunk_escaped()
+ */
+typedef enum {
+    TPL_ESCAPE_NONE = 0,
+    TPL_ESCAPE_HTML,
+    TPL_ESCAPE_JS,
+} tpl_escape_t;
+
+enum {
+    HTOPT_NONE = 0,
+    HTOPT_NO_HEADERS = 1 << 0,
+    HTOPT_NO_CLOSE = 1 << 1,
+    HTOPT_INCLUDE = HTOPT_NO_HEADERS|HTOPT_NO_CLOSE,
+};
+
+/**
+ * Send string using a given escaping scheme
+ *
+ * @param r
+ * @param buf - buf to send
+ * @param len - buf len, or HTTPD_RESP_USE_STRLEN
+ * @param escape - escaping type
+ * @return success
+ */
+esp_err_t httpd_resp_send_chunk_escaped(httpd_req_t *r, const char *buf, ssize_t len, tpl_escape_t escape);
+
+/**
+ * Template substitution callback. Data shall be sent using httpd_resp_send_chunk_escaped().
+ *
+ * @param[in,out] context - user-defined page state data
+ * @param[in] token - replacement token
+ * @return ESP_OK if the token was substituted, ESP_ERR_NOT_FOUND if it is unknown, other errors on e.g. send failure
+ */
+typedef esp_err_t (*template_subst_t)(httpd_req_t *r, void *context, const char *token, tpl_escape_t escape);
+
+/**
+ * Send a template file as a response. The content type from the file struct will be used.
+ *
+ * Use HTOPT_INCLUDE when used to embed a file inside a template.
+ *
+ * @param r - request
+ * @param file_index - file index in EMBEDDED_FILE_LOOKUP
+ * @param replacer - substitution callback, can be NULL if only includes are to be processed
+ * @param context - arbitrary context, will be passed to the replacer function; can be NULL
+ * @param opts - flag options (HTOPT_*)
+ */
+esp_err_t httpd_send_template_file(httpd_req_t *r, int file_index, template_subst_t replacer, void *context, uint32_t opts);
+
+/**
+ * Same as httpd_send_template_file, but using an `embedded_file_info` struct.
+ */
+esp_err_t httpd_send_template_file_struct(httpd_req_t *r, const struct embedded_file_info *file, template_subst_t replacer, void *context, uint32_t opts);
+
+/**
+ * Process and send a string template.
+ * The content-type header should be set beforehand, if different from the default (text/html).
+ *
+ * Use HTOPT_INCLUDE when used to embed a file inside a template.
+ *
+ * @param r - request
+ * @param template - template string (does not have to be terminated by a null byte)
+ * @param template_len - length of the template string; -1 to use strlen()
+ * @param replacer - substitution callback, can be NULL if only includes are to be processed
+ * @param context - arbitrary context, will be passed to the replacer function; can be NULL
+ * @param opts - flag options (HTOPT_*)
+ */
+esp_err_t httpd_send_template(httpd_req_t *r, const char *template, ssize_t template_len, template_subst_t replacer, void *context, uint32_t opts);
+
+/**
+ * Send a static file. This can be used to just send a file, or to embed a static template as a token substitution.
+ *
+ * Use HTOPT_INCLUDE when used to embed a file inside a template.
+ *
+ * Note: use httpd_resp_send_chunk_escaped() or httpd_resp_send_chunk() to send a plain string.
+ *
+ * @param r - request
+ * @param file_index - file index in EMBEDDED_FILE_LOOKUP
+ * @param escape - escape option
+ * @param opts - flag options (HTOPT_*)
+ * @return
+ */
+esp_err_t httpd_send_static_file(httpd_req_t *r, int file_index, tpl_escape_t escape, uint32_t opts);
+
+/**
+ * Same as httpd_send_template_file, but using an `embedded_file_info` struct.
+ */
+esp_err_t httpd_send_static_file_struct(httpd_req_t *r, const struct embedded_file_info *file, tpl_escape_t escape, uint32_t opts);
+
+struct tpl_kv_entry {
+    char key[TPL_KV_KEY_LEN]; // copied here
+    char subst[TPL_KV_SUBST_LEN]; // copied here
+    char *subst_heap;
+    SLIST_ENTRY(tpl_kv_entry) link;
+};
+
+SLIST_HEAD(tpl_kv_list, tpl_kv_entry);
+
+/**
+ * key-value replacer that works with a dynamically allocated SLIST.
+ *
+ * @param r - request
+ * @param context - context - must be a pointer to `struct tpl_kv_list`
+ * @param token - token to replace
+ * @param escape - escape option
+ * @return OK/not found/other
+ */
+esp_err_t tpl_kv_replacer(httpd_req_t *r, void *context, const char *token, tpl_escape_t escape);
+
+/**
+ * Add a pair into the substitutions list
+ *
+ * @param head - list head
+ * @param key - key, copied
+ * @param subst - value, copied
+ * @return success (fails if malloc failed)
+ */
+esp_err_t tpl_kv_add(struct tpl_kv_list *head, const char *key, const char *subst);
+
+/**
+ * Add a heap-allocated string to the replacer.
+ *
+ * @param head - list head
+ * @param key - key, copied
+ * @param subst - value, copied
+ * @return success (fails if malloc failed)
+ */
+esp_err_t tpl_kv_add_heapstr(struct tpl_kv_list *head, const char *key, char *subst);
+
+/**
+ * Convenience function that converts an IP address to string and adds it as a substitution
+ *
+ * @param head - list head
+ * @param key - key, copied
+ * @param ip4h - host order ipv4 address
+ * @return success
+ */
+esp_err_t tpl_kv_add_ipv4str(struct tpl_kv_list *head, const char *key, uint32_t ip4h);
+
+/** add int as a substitution; key is copied */
+esp_err_t tpl_kv_add_int(struct tpl_kv_list *head, const char *key, int32_t num);
+
+/** add long as a substitution; key is copied */
+esp_err_t tpl_kv_add_long(struct tpl_kv_list *head, const char *key, int64_t num);
+
+/** add printf-formatted value; key is copied */
+esp_err_t tpl_kv_sprintf(struct tpl_kv_list *head, const char *key, const char *format, ...)
+    __attribute__((format(printf,3,4)));
+
+/**
+ * Init a substitutions list (on the stack)
+ *
+ * @return the list
+ */
+static inline struct tpl_kv_list tpl_kv_init(void)
+{
+    return (struct tpl_kv_list) {.slh_first = NULL};
+}
+
+/**
+ * Free the list (head is left alone because it was allocated on the stack)
+ * @param head
+ */
+void tpl_kv_free(struct tpl_kv_list *head);
+
+/**
+ * Send the map as an ASCII table separated by Record Separator (30) and Unit Separator (31).
+ * Content type is set to application/octet-stream.
+ *
+ * key 31 value 30
+ * key 31 value 30
+ * key 31 value
+ *
+ * @param req
+ */
+esp_err_t tpl_kv_send_as_ascii_map(httpd_req_t *req, struct tpl_kv_list *head);
+
+#endif //FBNODE_TOKEN_SUBS_H

components/fileserver/readme/README.md

@@ -0,0 +1,29 @@
+Place the `rebuild_file_tables.php` script in a `files` subfolder of the main component.
+It requires PHP 7 to run. 
+
+This is what the setup should look like
+
+```
+main/files/embed/index.html
+main/files/rebuild_file_tables.php
+main/CMakeLists.txt
+main/main.c
+```
+
+Add this to your CMakeLists.txt before `register_component`:
+
+```
+#begin staticfiles
+#end staticfiles
+```
+
+The script will update CMakeLists.txt and generate `files_enum.c` and `files_enum.h` when run.
+
+```
+main/files/files_enum.h
+main/files/files_enum.c
+```
+
+Ensure `files/files_enum.c` is included in the build.
+
+`www_get_static_file()` is implemented as weak to let you provide custom access authentication logic.

components/fileserver/readme/rebuild_file_tables.php

@@ -0,0 +1,170 @@
+#!/usr/bin/env php
+<?php
+// This script rebuilds the static files enum, extern symbols pointing to the embedded byte buffers,
+// and the look-up structs table. To add more files, simply add them in the 'files' directory.
+
+// Note that all files will be accessible by the webserver, unless you filter them in embedded_files.c.
+
+
+// List all files
+$files = scandir(__DIR__.'/embed');
+
+$files = array_filter(array_map(function ($f) {
+  if (!is_file(__DIR__.'/embed/'.$f)) return null;
+  if (preg_match('/^\.|\.kate-swp|\.bak$|~$|\.sh$/', $f)) return null;
+
+  echo "Found: $f\n";
+  return $f;
+}, $files));
+
+sort($files);
+
+$formatted = array_filter(array_map(function ($f) {
+  return "\"files/embed/$f\"";
+}, $files));
+
+$cmake = file_get_contents(__DIR__.'/../CMakeLists.txt');
+
+$cmake = preg_replace('/#begin staticfiles\n.*#end staticfiles/s', 
+  "#begin staticfiles\n".
+  "set(COMPONENT_EMBED_FILES\n        ".
+  implode("\n        ", $formatted) . ")\n".
+  "#end staticfiles", 
+  $cmake);
+
+file_put_contents(__DIR__.'/../CMakeLists.txt', $cmake);
+
+
+// Generate a list of files
+
+$num = 0;
+$enum_keys = array_map(function ($f) use(&$num) {
+  $a = preg_replace("/[^A-Z0-9_]+/", "_", strtoupper($f));
+  return 'FILE_'. $a.' = '.($num++);
+}, $files);
+
+$keylist = implode(",\n    ", $enum_keys);
+
+$struct_array = [];
+
+$externs = array_map(function ($f) use (&$struct_array) {
+  $a = preg_replace("/[^A-Z0-9_]+/", "_", strtoupper($f));
+  
+  $start = '_binary_'. strtolower($a).'_start';
+  $end = '_binary_'. strtolower($a).'_end';
+  
+  static $mimes = array(
+    'txt' => 'text/plain',
+    'htm' => 'text/html',
+    'html' => 'text/html',
+    'php' => 'text/html',
+    'css' => 'text/css',
+    'js' => 'application/javascript',
+    'json' => 'application/json',
+    'xml' => 'application/xml',
+    'swf' => 'application/x-shockwave-flash',
+    'flv' => 'video/x-flv',
+    
+    'pem' => 'application/x-pem-file',
+
+    // images
+    'png' => 'image/png',
+    'jpe' => 'image/jpeg',
+    'jpeg' => 'image/jpeg',
+    'jpg' => 'image/jpeg',
+    'gif' => 'image/gif',
+    'bmp' => 'image/bmp',
+    'ico' => 'image/vnd.microsoft.icon',
+    'tiff' => 'image/tiff',
+    'tif' => 'image/tiff',
+    'svg' => 'image/svg+xml',
+    'svgz' => 'image/svg+xml',
+
+    // archives
+    'zip' => 'application/zip',
+    'rar' => 'application/x-rar-compressed',
+    'exe' => 'application/x-msdownload',
+    'msi' => 'application/x-msdownload',
+    'cab' => 'application/vnd.ms-cab-compressed',
+
+    // audio/video
+    'mp3' => 'audio/mpeg',
+    'qt' => 'video/quicktime',
+    'mov' => 'video/quicktime',
+
+    // adobe
+    'pdf' => 'application/pdf',
+    'psd' => 'image/vnd.adobe.photoshop',
+    'ai' => 'application/postscript',
+    'eps' => 'application/postscript',
+    'ps' => 'application/postscript',
+
+    // ms office
+    'doc' => 'application/msword',
+    'rtf' => 'application/rtf',
+    'xls' => 'application/vnd.ms-excel',
+    'ppt' => 'application/vnd.ms-powerpoint',
+
+    // open office
+    'odt' => 'application/vnd.oasis.opendocument.text',
+    'ods' => 'application/vnd.oasis.opendocument.spreadsheet',
+  );
+  
+  $parts = explode('.', $f);
+  $suffix = end($parts);
+  $mime = $mimes[$suffix] ?? 'application/octet-stream';
+  
+  $len = filesize('embed/'.$f);
+  
+  $struct_array[] = "[FILE_$a] = {{$start}, {$end}, \"{$f}\", \"{$mime}\"},";
+  
+  return 
+    'extern const uint8_t '.$start.'[];'."\n".
+    'extern const uint8_t '.$end.'[];';
+}, $files);
+
+$externlist = implode("\n", $externs);
+$structlist = implode("\n    ", $struct_array);
+
+
+file_put_contents('files_enum.h', <<<FILE
+// Do not change, auto-generated by gen_staticfiles.php
+
+#ifndef _EMBEDDED_FILES_ENUM_H
+#define _EMBEDDED_FILES_ENUM_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+enum embedded_file_id {
+    $keylist,
+    FILE_MAX
+};
+
+struct embedded_file_info {
+  const uint8_t * start;
+  const uint8_t * end;
+  const char * name;
+  const char * mime;
+};
+
+$externlist
+
+extern const struct embedded_file_info EMBEDDED_FILE_LOOKUP[];
+
+#endif // _EMBEDDED_FILES_ENUM_H
+
+FILE
+);
+
+file_put_contents("files_enum.c", <<<FILE
+// Do not change, auto-generated by gen_staticfiles.php
+#include <stdint.h>
+#include "files_enum.h"
+
+const struct embedded_file_info EMBEDDED_FILE_LOOKUP[] = {
+    $structlist
+};
+
+FILE
+);

components/fileserver/src/embedded_files.c

@@ -0,0 +1,22 @@
+#include <esp_err.h>
+#include "fileserver/embedded_files.h"
+#include "string.h"
+
+esp_err_t __attribute__((weak)) 
+www_get_static_file(const char *name, enum file_access_level access, const struct embedded_file_info **file)
+{
+    // simple search by name
+    for(int i = 0; i < EMBEDDED_FILE_LOOKUP_LEN; i++) {
+        if (0 == strcmp(EMBEDDED_FILE_LOOKUP[i].name, name)) {
+            *file = &EMBEDDED_FILE_LOOKUP[i];
+            return ESP_OK;
+        }
+    }
+
+    return ESP_ERR_NOT_FOUND;
+}
+
+bool __attribute__((weak)) 
+www_get_static_file_access_check(const struct embedded_file_info *file, enum file_access_level access) {
+  return true;
+}

components/fileserver/src/token_subs.c

@@ -0,0 +1,619 @@
+//#define LOG_LOCAL_LEVEL ESP_LOG_DEBUG
+
+#include <esp_log.h>
+#include <esp_http_server.h>
+#include <rom/queue.h>
+#include <lwip/ip4_addr.h>
+#include <sys/param.h>
+#include <common_utils/utils.h>
+#include <fileserver/token_subs.h>
+
+#include "fileserver/embedded_files.h"
+#include "fileserver/token_subs.h"
+
+#define ESP_TRY(x) \
+    do {  \
+        esp_err_t try_er = (x);  \
+        if (try_er != ESP_OK) return try_er;  \
+    } while(0)
+
+static const char* TAG = "token_subs";
+
+// TODO implement buffering to avoid sending many tiny chunks when escaping
+
+/* encode for HTML. returns 0 or 1 - 1 = success */
+static esp_err_t send_html_chunk(httpd_req_t *r, const char *data, ssize_t len)
+{
+    assert(r);
+    assert(data);
+
+    int start = 0, end = 0;
+    char c;
+    if (len < 0) len = (int) strlen(data);
+    if (len==0) return ESP_OK;
+
+    for (end = 0; end < len; end++) {
+        c = data[end];
+        if (c == 0) {
+            // we found EOS
+            break; // not return - the last chunk is printed after the loop
+        }
+
+        if (c == '"' || c == '\'' || c == '<' || c == '>') {
+            if (start < end) ESP_TRY(httpd_resp_send_chunk(r, data + start, end - start));
+            start = end + 1;
+        }
+
+        if (c == '"') ESP_TRY(httpd_resp_send_chunk(r, "&#34;", 5));
+        else if (c == '\'') ESP_TRY(httpd_resp_send_chunk(r, "&#39;", 5));
+        else if (c == '<') ESP_TRY(httpd_resp_send_chunk(r, "&lt;", 4));
+        else if (c == '>') ESP_TRY(httpd_resp_send_chunk(r, "&gt;", 4));
+    }
+
+    if (start < end) ESP_TRY(httpd_resp_send_chunk(r, data + start, end - start));
+    return ESP_OK;
+}
+
+/* encode for JS. returns 0 or 1 - 1 = success */
+static esp_err_t send_js_chunk(httpd_req_t *r, const char *data, ssize_t len)
+{
+    assert(r);
+    assert(data);
+
+    int start = 0, end = 0;
+    char c;
+    if (len < 0) len = (int) strlen(data);
+    if (len==0) return ESP_OK;
+
+    for (end = 0; end < len; end++) {
+        c = data[end];
+        if (c == 0) {
+            // we found EOS
+            break; // not return - the last chunk is printed after the loop
+        }
+
+        if (c == '"' || c == '\\' || c == '/' || c == '\'' || c == '<' || c == '>' || c == '\n' || c == '\r') {
+            if (start < end) ESP_TRY(httpd_resp_send_chunk(r, data + start, end - start));
+            start = end + 1;
+        }
+
+        if (c == '"') ESP_TRY(httpd_resp_send_chunk(r, "\\\"", 2));
+        else if (c == '\'') ESP_TRY(httpd_resp_send_chunk(r, "\\'", 2));
+        else if (c == '\\') ESP_TRY(httpd_resp_send_chunk(r, "\\\\", 2));
+        else if (c == '/') ESP_TRY(httpd_resp_send_chunk(r, "\\/", 2));
+        else if (c == '<') ESP_TRY(httpd_resp_send_chunk(r, "\\u003C", 6));
+        else if (c == '>') ESP_TRY(httpd_resp_send_chunk(r, "\\u003E", 6));
+        else if (c == '\n') ESP_TRY(httpd_resp_send_chunk(r, "\\n", 2));
+        else if (c == '\r') ESP_TRY(httpd_resp_send_chunk(r, "\\r", 2));
+    }
+
+    if (start < end) ESP_TRY(httpd_resp_send_chunk(r, data + start, end - start));
+    return ESP_OK;
+}
+
+
+esp_err_t httpd_resp_send_chunk_escaped(httpd_req_t *r, const char *buf, ssize_t len, tpl_escape_t escape)
+{
+    switch (escape) {
+        default: // this enum should be exhaustive, but in case something went wrong, just print it verbatim
+
+        case TPL_ESCAPE_NONE:
+            return httpd_resp_send_chunk(r, buf, len);
+
+        case TPL_ESCAPE_HTML:
+            return send_html_chunk(r, buf, len);
+
+        case TPL_ESCAPE_JS:
+            return send_js_chunk(r, buf, len);
+    }
+}
+
+esp_err_t httpd_send_static_file(httpd_req_t *r, int file_index, tpl_escape_t escape, uint32_t opts)
+{
+    assert(file_index < EMBEDDED_FILE_LOOKUP_LEN);
+    const struct embedded_file_info *file = &EMBEDDED_FILE_LOOKUP[file_index];
+
+    return httpd_send_static_file_struct(r, file, escape, opts);
+}
+
+esp_err_t httpd_send_static_file_struct(httpd_req_t *r, const struct embedded_file_info *file, tpl_escape_t escape, uint32_t opts)
+{
+    if (0 == (opts & HTOPT_NO_HEADERS)) {
+        ESP_TRY(httpd_resp_set_type(r, file->mime));
+        ESP_TRY(httpd_resp_set_hdr(r, "Cache-Control", "max-age=86400, public, must-revalidate"));
+    }
+
+    ESP_TRY(httpd_resp_send_chunk_escaped(r, (const char *) file->start, (size_t)(file->end - file->start), escape));
+
+    if (0 == (opts & HTOPT_NO_CLOSE)) {
+        ESP_TRY(httpd_resp_send_chunk(r, NULL, 0));
+    }
+
+    return ESP_OK;
+}
+
+esp_err_t httpd_send_template_file(httpd_req_t *r,
+                                   int file_index,
+                                   template_subst_t replacer,
+                                   void *context,
+                                   uint32_t opts)
+{
+    assert(file_index < EMBEDDED_FILE_LOOKUP_LEN);
+    const struct embedded_file_info *file = &EMBEDDED_FILE_LOOKUP[file_index];
+    return httpd_send_template_file_struct(r,file,replacer,context,opts);
+}
+
+esp_err_t httpd_send_template_file_struct(httpd_req_t *r,
+                                   const struct embedded_file_info *file,
+                                   template_subst_t replacer,
+                                   void *context,
+                                   uint32_t opts)
+{
+    if (0 == (opts & HTOPT_NO_HEADERS)) {
+        ESP_TRY(httpd_resp_set_type(r, file->mime));
+        ESP_TRY(httpd_resp_set_hdr(r, "Cache-Control", "no-cache, no-store, must-revalidate"));
+    }
+
+    return httpd_send_template(r, (const char *) file->start, (size_t)(file->end - file->start), replacer, context, opts);
+}
+
+esp_err_t tpl_kv_replacer(httpd_req_t *r, void *context, const char *token, tpl_escape_t escape)
+{
+    assert(context);
+    assert(token);
+
+    struct tpl_kv_entry *entry;
+    struct tpl_kv_list *head = context;
+    SLIST_FOREACH(entry, head, link) {
+        if (0==strcmp(entry->key, token)) {
+            return httpd_resp_send_chunk_escaped(r,
+                entry->subst_heap ?
+                    entry->subst_heap :
+                    entry->subst,
+                -1, escape);
+        }
+    }
+
+    return ESP_ERR_NOT_FOUND;
+}
+
+struct stacked_replacer_context {
+    template_subst_t replacer0;
+    void *context0;
+    template_subst_t replacer1;
+    void *context1;
+};
+
+esp_err_t stacked_replacer(httpd_req_t *r, void *context, const char *token, tpl_escape_t escape)
+{
+    assert(context);
+    assert(token);
+
+    struct stacked_replacer_context *combo = context;
+
+    if (ESP_OK == combo->replacer0(r, combo->context0, token, escape)) {
+        return ESP_OK;
+    }
+
+    if (ESP_OK == combo->replacer1(r, combo->context1, token, escape)) {
+        return ESP_OK;
+    }
+
+    return ESP_ERR_NOT_FOUND;
+}
+
+esp_err_t httpd_send_template(httpd_req_t *r,
+                              const char *template, ssize_t template_len,
+                              template_subst_t replacer,
+                              void *context,
+                              uint32_t opts)
+{
+    if (template_len < 0) template_len = strlen(template);
+
+    // replacer and context may be NULL
+    assert(template);
+    assert(r);
+
+    // data end
+    const char * const end = template + template_len;
+
+    // start of to-be-processed data
+    const char * pos = template;
+
+    // start position for finding opening braces, updated after a failed match to avoid infinite loop on the same bad token
+    const char * searchpos = pos;
+
+    // tokens must be copied to a buffer to allow adding the terminating null byte
+    char token_buf[MAX_TOKEN_LEN];
+
+    while (pos < end) {
+        const char * openbr = strchr(searchpos, '{');
+        if (openbr == NULL) {
+            // no more templates
+            ESP_TRY(httpd_resp_send_chunk(r, pos, (size_t) (end - pos)));
+            break;
+        }
+
+        // this brace could start a valid template. check if it seems valid...
+        const char * closebr = strchr(openbr, '}');
+        if (closebr == NULL) {
+            // there are no further closing braces, so it can't be a template
+
+            // we also know there can't be any more substitutions, because they would lack a closing } too
+            ESP_TRY(httpd_resp_send_chunk(r, pos, (size_t) (end - pos)));
+            break;
+        }
+
+        // see if the braces content looks like a token
+        const char *t = openbr + 1;
+        bool token_valid = true;
+        struct tpl_kv_list substitutions_head = tpl_kv_init();
+        struct tpl_kv_entry *new_subst_pair = NULL;
+
+        // a token can be either a name for replacement by the replacer func, or an include with static kv replacements
+        bool is_include = false;
+        bool token_is_optional = false;
+        const char *token_end = NULL; // points one char after the end of the token
+
+        // parsing the token
+        {
+            if (*t == '@') {
+                ESP_LOGD(TAG, "Parsing an Include token");
+                is_include = true;
+                t++;
+            }
+
+            enum {
+                P_NAME, P_KEY, P_VALUE
+            } state = P_NAME;
+
+            const char *kv_start = NULL;
+            while (t != closebr || state == P_VALUE) {
+                char c = *t;
+
+                if (state == P_NAME) {
+                    if (!((c >= 'a' && c <= 'z') ||
+                          (c >= 'A' && c <= 'Z') ||
+                          (c >= '0' && c <= '9') ||
+                          c == '.' || c == '_' || c == '-' || c == ':')) {
+
+                        if (!is_include && c == '?') {
+                            token_end = t;
+                            token_is_optional = true;
+                        } else {
+                            if (is_include && c == '|') {
+                                token_end = t;
+                                state = P_KEY;
+                                kv_start = t + 1;
+                                // pipe separates the include's filename and literal substitutions
+                                // we know there is a closing } somewhere, and {@....| doesn't occur normally, so let's assume it's correct
+                            }
+                            else {
+                                token_valid = false;
+                                break;
+                            }
+                        }
+                    }
+                }
+                else if (state == P_KEY) {
+                    if (!((c >= 'a' && c <= 'z') ||
+                          (c >= 'A' && c <= 'Z') ||
+                          (c >= '0' && c <= '9') ||
+                          c == '.' || c == '_' || c == '-')) {
+                        if (c == '=') {
+                            new_subst_pair = calloc(1, sizeof(struct tpl_kv_entry));
+                            const size_t klen = MIN(TPL_KV_KEY_LEN, t - kv_start);
+                            strncpy(new_subst_pair->key, kv_start, klen);
+                            new_subst_pair->key[klen] = 0;
+
+                            kv_start = t + 1;
+
+                            state = P_VALUE;
+                            // pipe separates the include's filename and literal substitutions
+                            // we know there is a closing } somewhere, and {@....| doesn't occur normally, so let's assume it's correct
+                        }
+                    }
+                }
+                else if (state == P_VALUE) {
+                    if (c == '|' || c == '}') {
+                        const size_t vlen = MIN(TPL_KV_SUBST_LEN, t - kv_start);
+                        strncpy(new_subst_pair->subst, kv_start, vlen);
+                        new_subst_pair->subst[vlen] = 0;
+
+                        // attach the kv pair to the list
+                        SLIST_INSERT_HEAD(&substitutions_head, new_subst_pair, link);
+                        ESP_LOGD(TAG, "Adding subs kv %s -> %s", new_subst_pair->key, new_subst_pair->subst);
+                        new_subst_pair = NULL;
+
+                        kv_start = t + 1; // go past the pipe
+                        state = P_KEY;
+
+                        if (t == closebr) {
+                            break; // found the ending brace, so let's quit the kv parse loop
+                        }
+                    }
+                }
+
+                t++;
+            }
+            // clean up after a messed up subs kv pairs syntax
+            if (new_subst_pair != NULL) {
+                free(new_subst_pair);
+            }
+        }
+
+        if (!token_valid) {
+            // false match, include it in the block to send before the next token
+            searchpos = openbr + 1;
+            ESP_LOGD(TAG, "Skip invalid token near %10s", openbr);
+            continue;
+        }
+
+        // now we know it looks like a substitution token
+
+        // flush data before the token
+        if (pos != openbr) ESP_TRY(httpd_resp_send_chunk(r, pos, (size_t) (openbr - pos)));
+
+        const char *token_start = openbr;
+
+        tpl_escape_t escape = TPL_ESCAPE_NONE;
+
+        // extract and terminate the token
+        size_t token_len = MIN(MAX_TOKEN_LEN-1, closebr - openbr - 1);
+        if (token_end) {
+            token_len = MIN(token_len, token_end - openbr - 1);
+        }
+
+        if (is_include) {
+            token_start += 1; // skip the @
+            token_len -= 1;
+        } else {
+            if (0 == strncmp("h:", openbr + 1, 2)) {
+                escape = TPL_ESCAPE_HTML;
+                token_start += 2;
+                token_len -= 2;
+            }
+            else if (0 == strncmp("j:", openbr + 1, 2)) {
+                escape = TPL_ESCAPE_JS;
+                token_start += 2;
+                token_len -= 2;
+            }
+        }
+
+        strncpy(token_buf, token_start+1, token_len);
+        token_buf[token_len] = 0;
+
+        ESP_LOGD(TAG, "Token: %s", token_buf);
+
+        esp_err_t rv;
+
+        if (is_include) {
+            ESP_LOGD(TAG, "Trying to include a sub-file");
+
+            const struct embedded_file_info *file = NULL;
+            rv = www_get_static_file(token_buf, FILE_ACCESS_PROTECTED, &file);
+            if (rv != ESP_OK) {
+                ESP_LOGE(TAG, "Failed to statically include \"%s\" in a template - %s", token_buf, esp_err_to_name(rv));
+                // this will cause the token to be emitted verbatim
+            } else {
+                ESP_LOGD(TAG, "Descending...");
+
+                // combine the two replacers
+                struct stacked_replacer_context combo = {
+                    .replacer0 = replacer,
+                    .context0 = context,
+                    .replacer1 = tpl_kv_replacer,
+                    .context1 = &substitutions_head
+                };
+
+                rv = httpd_send_template_file_struct(r, file, stacked_replacer, &combo, HTOPT_INCLUDE);
+                ESP_LOGD(TAG, "...back in parent");
+            }
+
+            // tear down the list
+            tpl_kv_free(&substitutions_head);
+
+            if (rv != ESP_OK) {
+                // just send it verbatim...
+                ESP_TRY(httpd_resp_send_chunk(r, openbr, (size_t) (closebr - openbr + 1)));
+            }
+        } else {
+            if (replacer) {
+                ESP_LOGD(TAG, "Running replacer for \"%s\" with escape %d", token_buf, escape);
+                rv = replacer(r, context, token_buf, escape);
+
+                if (rv != ESP_OK) {
+                    if (rv == ESP_ERR_NOT_FOUND) {
+                        ESP_LOGD(TAG, "Token rejected");
+                        // optional token becomes empty string if not replaced
+                        if (!token_is_optional) {
+                            ESP_LOGD(TAG, "Not optional, keeping verbatim");
+                            // replacer rejected the token, keep it verbatim
+                            ESP_TRY(httpd_resp_send_chunk(r, openbr, (size_t) (closebr - openbr + 1)));
+                        }
+                    }
+                    else {
+                        ESP_LOGE(TAG, "Unexpected error from replacer func: 0x%02x - %s", rv, esp_err_to_name(rv));
+                        return rv;
+                    }
+                }
+            } else {
+                ESP_LOGD(TAG, "Not replacer");
+                // no replacer, only includes - used for 'static' files
+                if (!token_is_optional) {
+                    ESP_LOGD(TAG, "Token not optional, keeping verbatim");
+                    ESP_TRY(httpd_resp_send_chunk(r, openbr, (size_t) (closebr - openbr + 1)));
+                }
+            }
+        }
+
+        searchpos = pos = closebr + 1;
+    }
+
+    if (0 == (opts & HTOPT_NO_CLOSE)) {
+        ESP_TRY(httpd_resp_send_chunk(r, NULL, 0));
+    }
+
+    return ESP_OK;
+}
+
+
+esp_err_t tpl_kv_add_int(struct tpl_kv_list *head, const char *key, int32_t num)
+{
+    char buf[12];
+    itoa(num, buf, 10);
+    return tpl_kv_add(head, key, buf);
+}
+
+esp_err_t tpl_kv_add_long(struct tpl_kv_list *head, const char *key, int64_t num)
+{
+    char buf[21];
+    sprintf(buf, "%"PRIi64, num);
+    return tpl_kv_add(head, key, buf);
+}
+
+esp_err_t tpl_kv_add_ipv4str(struct tpl_kv_list *head, const char *key, uint32_t ip4h)
+{
+    char buf[IP4ADDR_STRLEN_MAX];
+    ip4_addr_t addr;
+    addr.addr = lwip_htonl(ip4h);
+    ip4addr_ntoa_r(&addr, buf, IP4ADDR_STRLEN_MAX);
+
+    return tpl_kv_add(head, key, buf);
+}
+
+esp_err_t tpl_kv_add(struct tpl_kv_list *head, const char *key, const char *subst)
+{
+    ESP_LOGD(TAG, "kv add subs %s := %s", key, subst);
+    struct tpl_kv_entry *entry = calloc(1, sizeof(struct tpl_kv_entry));
+    if (entry == NULL) return ESP_ERR_NO_MEM;
+
+    assert(strlen(key) < TPL_KV_KEY_LEN);
+    assert(strlen(subst) < TPL_KV_SUBST_LEN);
+
+    strncpy(entry->key, key, TPL_KV_KEY_LEN);
+    entry->key[TPL_KV_KEY_LEN - 1] = 0;
+
+    strncpy(entry->subst, subst, TPL_KV_SUBST_LEN - 1);
+    entry->subst[TPL_KV_KEY_LEN - 1] = 0;
+
+    SLIST_INSERT_HEAD(head, entry, link);
+    return ESP_OK;
+}
+
+esp_err_t tpl_kv_add_heapstr(struct tpl_kv_list *head, const char *key, char *subst)
+{
+    ESP_LOGD(TAG, "kv add subs %s := (heap str)", key);
+    struct tpl_kv_entry *entry = calloc(1, sizeof(struct tpl_kv_entry));
+    if (entry == NULL) return ESP_ERR_NO_MEM;
+
+    assert(strlen(key) < TPL_KV_KEY_LEN);
+
+    strncpy(entry->key, key, TPL_KV_KEY_LEN);
+    entry->key[TPL_KV_KEY_LEN - 1] = 0;
+
+    entry->subst_heap = subst;
+
+    SLIST_INSERT_HEAD(head, entry, link);
+    return ESP_OK;
+}
+
+esp_err_t tpl_kv_sprintf(struct tpl_kv_list *head, const char *key, const char *format, ...)
+{
+    ESP_LOGD(TAG, "kv printf %s := %s", key, format);
+    struct tpl_kv_entry *entry = calloc(1, sizeof(struct tpl_kv_entry));
+    if (entry == NULL) return ESP_ERR_NO_MEM;
+
+    assert(strlen(key) < TPL_KV_KEY_LEN);
+
+    strncpy(entry->key, key, TPL_KV_KEY_LEN);
+    entry->key[TPL_KV_KEY_LEN - 1] = 0;
+
+    va_list list;
+    va_start(list, format);
+    vsnprintf(entry->subst, TPL_KV_SUBST_LEN, format, list);
+    va_end(list);
+    entry->subst[TPL_KV_KEY_LEN - 1] = 0;
+
+    SLIST_INSERT_HEAD(head, entry, link);
+    return ESP_OK;
+}
+
+void tpl_kv_free(struct tpl_kv_list *head)
+{
+    struct tpl_kv_entry *item, *next;
+    SLIST_FOREACH_SAFE(item, head, link, next) {
+        if (item->subst_heap) {
+            free(item->subst_heap);
+            item->subst_heap = NULL;
+        }
+
+        free(item);
+    }
+}
+
+esp_err_t tpl_kv_send_as_ascii_map(httpd_req_t *req, struct tpl_kv_list *head)
+{
+    httpd_resp_set_type(req, "text/plain; charset=utf-8");
+
+#define BUF_CAP 512
+    char *buf_head = malloc(BUF_CAP);
+    if (!buf_head) {
+        ESP_LOGE(TAG, "Malloc err");
+        return ESP_FAIL;
+    }
+    char *buf = buf_head;
+    size_t cap = BUF_CAP;
+    struct tpl_kv_entry *entry;
+
+    // GCC nested function
+    esp_err_t send_part() {
+        esp_err_t suc = httpd_resp_send_chunk(req, buf_head, BUF_CAP-cap);
+        buf = buf_head; // buf is assigned to buf head
+        cap = BUF_CAP;
+        if (suc != ESP_OK) {
+            ESP_LOGE(TAG, "Error sending buffer");
+            free(buf_head);
+            httpd_resp_send_chunk(req, NULL, 0);
+        }
+        return suc;
+    }
+
+    SLIST_FOREACH(entry, head, link) {
+        while(NULL == (buf = append(buf, entry->key, &cap))) ESP_TRY(send_part());
+        while(NULL == (buf = append(buf, "\x1f", &cap))) ESP_TRY(send_part());
+
+        if (entry->subst_heap) {
+            if (strlen(entry->subst_heap) >= BUF_CAP) {
+                // send what we have
+                ESP_TRY(send_part());
+                esp_err_t suc = httpd_resp_send_chunk(req, entry->subst_heap, -1);
+                if (suc != ESP_OK) {
+                    ESP_LOGE(TAG, "Error sending buffer");
+                    free(buf_head);
+                    httpd_resp_send_chunk(req, NULL, 0);
+                }
+            } else {
+                while (NULL == (buf = append(buf, entry->subst_heap, &cap))) ESP_TRY(send_part());
+            }
+        } else {
+            while(NULL == (buf = append(buf, entry->subst, &cap))) ESP_TRY(send_part());
+        }
+
+        if (entry->link.sle_next) {
+            while(NULL == (buf = append(buf, "\x1e", &cap))) ESP_TRY(send_part());
+        }
+    }
+    // send leftovers
+    if (buf != buf_head) {
+        esp_err_t suc = httpd_resp_send_chunk(req, buf_head, BUF_CAP-cap);
+        if (suc != ESP_OK) {
+            ESP_LOGE(TAG, "Error sending buffer");
+        }
+    }
+
+    // Commit
+    httpd_resp_send_chunk(req, NULL, 0);
+    free(buf_head);
+    return ESP_OK;
+}