working unicode!!! WTF!! and ajax initial load to avoid having to escape shit

html_orig/js/app.js

@@ -1091,7 +1091,6 @@ function tr(key) { return _tr[key] || '?'+key+'?'; }
 			ap.enc = parseInt(ap.enc);
 
 			if (ap.enc > 4) return; // hide unsupported auths
-			WiFi.scan_url = '/cfg/wifi/scan';
 
 			var item = mk('div');
 
@@ -1145,7 +1144,7 @@ function tr(key) { return _tr[key] || '?'+key+'?'; }
 
 	/** Ask the CGI what APs are visible (async) */
 	function scanAPs() {
-		$.get('http://'+_root+w.scan_url, onScan);
+		$.get('http://'+_root+'/cfg/wifi/scan', onScan);
 	}
 
 	function rescan(time) {
@@ -1183,7 +1182,7 @@ function tr(key) { return _tr[key] || '?'+key+'?'; }
 	w.startScanning = startScanning;
 })(window.WiFi = {});
 var Screen = (function () {
-	var W, H; // dimensions
+	var W = 0, H = 0; // dimensions
 	var inited = false;
 
 	var cursor = {
@@ -1338,21 +1337,21 @@ var Screen = (function () {
 		if (!inited) _init();
 
 		// Set size
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2;  // height
-		num2 = parse2B(str, i); i += 2;
+		num2 = parse2B(str, i); i += 2; // width
 		if (num != H || num2 != W) {
 			_rebuild(num, num2);
 		}
 		console.log("Size ",num, num2);
 
 		// Cursor position
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2; // row
-		num2 = parse2B(str, i); i += 2;
+		num2 = parse2B(str, i); i += 2; // col
 		cursorSet(num, num2);
 		console.log("Cursor at ",num, num2);
 
 		// Attributes
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2; // fg bg bold hidden
 		cursor.fg = num & 0x0F;
 		cursor.bg = (num & 0xF0) >> 4;
 		cursor.bold = !!(num & 0x100);
@@ -1444,12 +1443,19 @@ var Conn = (function() {
 	}
 
 	function init() {
-		ws = new WebSocket("ws://"+_root+"/ws/update.cgi");
+		ws = new WebSocket("ws://"+_root+"/term/update.ws");
 		ws.onopen = onOpen;
 		ws.onclose = onClose;
 		ws.onmessage = onMessage;
 
 		console.log("Opening socket.");
+
+		// Ask for initial data
+		$.get('http://'+_root+'/term/init', function(resp, status) {
+			if (status !== 200) location.reload(true);
+			console.log("Data received!");
+			Screen.load(resp);
+		});
 	}
 
 	return {
@@ -1511,8 +1517,7 @@ var Input = (function() {
 	};
 })();
 
-window.termInit = function (str) {
+window.termInit = function () {
-	Screen.load(str);
 	Conn.init();
 	Input.init();
 };

html_orig/jssrc/term.js

@@ -1,5 +1,5 @@
 var Screen = (function () {
-	var W, H; // dimensions
+	var W = 0, H = 0; // dimensions
 	var inited = false;
 
 	var cursor = {
@@ -154,21 +154,21 @@ var Screen = (function () {
 		if (!inited) _init();
 
 		// Set size
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2;  // height
-		num2 = parse2B(str, i); i += 2;
+		num2 = parse2B(str, i); i += 2; // width
 		if (num != H || num2 != W) {
 			_rebuild(num, num2);
 		}
 		console.log("Size ",num, num2);
 
 		// Cursor position
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2; // row
-		num2 = parse2B(str, i); i += 2;
+		num2 = parse2B(str, i); i += 2; // col
 		cursorSet(num, num2);
 		console.log("Cursor at ",num, num2);
 
 		// Attributes
-		num = parse2B(str, i); i += 2;
+		num = parse2B(str, i); i += 2; // fg bg bold hidden
 		cursor.fg = num & 0x0F;
 		cursor.bg = (num & 0xF0) >> 4;
 		cursor.bold = !!(num & 0x100);
@@ -260,12 +260,19 @@ var Conn = (function() {
 	}
 
 	function init() {
-		ws = new WebSocket("ws://"+_root+"/ws/update.cgi");
+		ws = new WebSocket("ws://"+_root+"/term/update.ws");
 		ws.onopen = onOpen;
 		ws.onclose = onClose;
 		ws.onmessage = onMessage;
 
 		console.log("Opening socket.");
+
+		// Ask for initial data
+		$.get('http://'+_root+'/term/init', function(resp, status) {
+			if (status !== 200) location.reload(true);
+			console.log("Data received!");
+			Screen.load(resp);
+		});
 	}
 
 	return {
@@ -327,8 +334,7 @@ var Input = (function() {
 	};
 })();
 
-window.termInit = function (str) {
+window.termInit = function () {
-	Screen.load(str);
 	Conn.init();
 	Input.init();
 };

html_orig/jssrc/wifi.js

@@ -68,7 +68,6 @@
 			ap.enc = parseInt(ap.enc);
 
 			if (ap.enc > 4) return; // hide unsupported auths
-			WiFi.scan_url = '/cfg/wifi/scan';
 
 			var item = mk('div');
 
@@ -122,7 +121,7 @@
 
 	/** Ask the CGI what APs are visible (async) */
 	function scanAPs() {
-		$.get('http://'+_root+w.scan_url, onScan);
+		$.get('http://'+_root+'/cfg/wifi/scan', onScan);
 	}
 
 	function rescan(time) {

html_orig/pages/term.php

@@ -22,7 +22,7 @@
 	</div>
 </div>
 
-<input id="softkb-input" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false">
+<textarea id="softkb-input" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"></textarea>
 
 <nav id="botnav">
 	<a href="#" onclick="toggleSoftKb(true); return false" class="icn-keyboard mq-tablet-max"></a><!--
@@ -34,7 +34,7 @@
 <script>
 	// TODO cleanup
 	try {
-		termInit("%screenData%");
+		termInit();
 
 		// auto-clear the input box
 		$('#softkb-input').on('input', function(e) {
@@ -47,7 +47,8 @@
 	}
 
 	function toggleSoftKb(yes) {
-		qs('#softkb-input')[yes ? 'focus' : 'blur']();
+		var i = qs('#softkb-input');
-		qs('.icn-keyboard').blur();
+		if (yes) i.focus();
+		else i.blur();
 	}
 </script>

user/ansi_parser_callbacks.c

@@ -10,13 +10,47 @@
 #include "ansi_parser.h"
 #include "uart_driver.h"
 
+static char utf_collect[4];
+static int utf_i = 0;
+static int utf_j = 0;
+
 /**
  * Handle a received plain character
  */
 void ICACHE_FLASH_ATTR
 apars_handle_plainchar(char c)
 {
-	screen_putchar(c);
+	// collecting unicode glyphs...
+	if (c & 0x80) {
+		if (utf_i == 0) {
+			if ((c & 0xE0) == 0xC0) {
+				utf_i = 2;
+			}
+			else if ((c & 0xF0) == 0xE0) {
+				utf_i = 3;
+			}
+			else if ((c & 0xF8) == 0xF0) {
+				utf_i = 4;
+			}
+
+			utf_collect[0] = c;
+			utf_j = 1;
+		}
+		else {
+			utf_collect[utf_j++] = c;
+			if (utf_j >= utf_i) {
+				screen_putchar(utf_collect);
+				utf_i = 0;
+				utf_j = 0;
+				memset(utf_collect, 0, 4);
+			}
+		}
+	}
+	else {
+		utf_collect[0] = c;
+		utf_collect[1] = 0; // just to make sure it's closed...
+		screen_putchar(utf_collect);
+	}
 }
 
 /**
@@ -188,8 +222,7 @@ apars_handle_CSI(char leadchar, int *params, char keychar)
 				int n = params[i];
 
 				if (i == 0 && n == 0) { // reset SGR
-					screen_set_fg(7);
+					screen_reset_cursor();
-					screen_set_bg(0);
 					break; // cannot combine reset with others
 				}
 				else if (n >= 30 && n <= 37) screen_set_fg(n-30); // ANSI normal fg

user/cgi_main.c

@@ -19,12 +19,10 @@ httpd_cgi_state ICACHE_FLASH_ATTR tplScreen(HttpdConnData *connData, char *token
 {
 	if (token == NULL) {
 		// Release data object
-		screenSerializeToBuffer(NULL, 0, arg);
 		return HTTPD_CGI_DONE;
 	}
 
-	const int bufsiz = 512;
+	char buff[100];
-	char buff[bufsiz];
 
 	if (streq(token, "term_title")) {
 		httpdSend(connData, termconf->title, -1);
@@ -44,29 +42,41 @@ httpd_cgi_state ICACHE_FLASH_ATTR tplScreen(HttpdConnData *connData, char *token
 	else if (streq(token, "btn5")) {
 		httpdSend(connData, termconf->btn5, -1);
 	}
-//	else if (streq(token, "default_bg")) {
-//		sprintf(buff, "%d", termconf->default_bg);
-//		httpdSend(connData, buff, -1);
-//	}
-//	else if (streq(token, "default_fg")) {
-//		sprintf(buff, "%d", termconf->default_fg);
-//		httpdSend(connData, buff, -1);
-//	}
 	else if (streq(token, "theme")) {
 		sprintf(buff, "%d", termconf->theme);
 		httpdSend(connData, buff, -1);
 	}
-	else if (streq(token, "screenData")) {
-		httpd_cgi_state cont = screenSerializeToBuffer(buff, bufsiz, arg);
-		httpdSend(connData, buff, -1);
-		return cont;
-	}
 
 	return HTTPD_CGI_DONE;
 }
 
+httpd_cgi_state ICACHE_FLASH_ATTR
+cgiTermInitialImage(HttpdConnData *connData)
+{
+	const int bufsiz = 512;
+	char buff[bufsiz];
+
+	if (connData->conn == NULL) {
+		//Connection aborted. Clean up.
+		// Release data object
+		screenSerializeToBuffer(NULL, 0, &connData->cgiData);
+		return HTTPD_CGI_DONE;
+	}
+
+	if (connData->cgiData == NULL) {
+		httpdStartResponse(connData, 200);
+		httpdHeader(connData, "Content-Type", "application/octet-stream");
+		httpdEndHeaders(connData);
+	}
+
+	httpd_cgi_state cont = screenSerializeToBuffer(buff, bufsiz, &connData->cgiData);
+	httpdSend(connData, buff, -1);
+	return cont;
+}
+
 /** "About" page */
-httpd_cgi_state ICACHE_FLASH_ATTR tplAbout(HttpdConnData *connData, char *token, void **arg)
+httpd_cgi_state ICACHE_FLASH_ATTR
+tplAbout(HttpdConnData *connData, char *token, void **arg)
 {
 	if (token == NULL) return HTTPD_CGI_DONE;
 

user/cgi_main.h

@@ -3,5 +3,6 @@
 
 httpd_cgi_state tplScreen(HttpdConnData *connData, char *token, void **arg);
 httpd_cgi_state tplAbout(HttpdConnData *connData, char *token, void **arg);
+httpd_cgi_state cgiTermInitialImage(HttpdConnData *connData);
 
 #endif // CGI_MAIN_H

user/cgi_sockets.h

@@ -1,7 +1,7 @@
 #ifndef CGI_SOCKETS_H
 #define CGI_SOCKETS_H
 
-#define URL_WS_UPDATE "/ws/update.cgi"
+#define URL_WS_UPDATE "/term/update.ws"
 
 /** Update websocket connect callback */
 void updateSockConnect(Websock *ws);

user/routes.c

@@ -32,6 +32,7 @@ HttpdBuiltInUrl routes[] = {
 	ROUTE_FILE("/help/?", "/help.tpl"),
 
 	// --- Sockets ---
+	ROUTE_CGI("/term/init", cgiTermInitialImage),
 	ROUTE_WS(URL_WS_UPDATE, updateSockConnect),
 
 	// --- System control ---

user/screen.c

@@ -46,9 +46,10 @@ void terminal_apply_settings(void)
  * Screen cell data type (16 bits)
  */
 typedef struct __attribute__((packed)){
-	char c   : 8;
+	char c[4]; // space for a full unicode character
 	Color fg : 4;
 	Color bg : 4;
+	bool bold : 1;
 } Cell;
 
 /**
@@ -64,7 +65,8 @@ static struct {
 	int y;    //!< Y coordinate
 	bool visible;    //!< Visible
 	bool inverse;    //!< Inverse colors
-	bool autowrap;    //!< Wrapping when EOL
+	bool autowrap;   //!< Wrapping when EOL
+	bool bold;       //!< Bold style
 	Color fg;        //!< Foreground color for writing
 	Color bg;        //!< Background color for writing
 } cursor;
@@ -108,7 +110,10 @@ clear_range(unsigned int from, unsigned int to)
 	Color fg = cursor.inverse ? cursor.bg : cursor.fg;
 	Color bg = cursor.inverse ? cursor.fg : cursor.bg;
 	for (unsigned int i = from; i <= to; i++) {
-		screen[i].c = ' ';
+		screen[i].c[0] = ' ';
+		screen[i].c[1] = 0;
+		screen[i].c[2] = 0;
+		screen[i].c[3] = 0;
 		screen[i].fg = fg;
 		screen[i].bg = bg;
 	}
@@ -127,6 +132,7 @@ cursor_reset(void)
 	cursor.visible = 1;
 	cursor.inverse = 0;
 	cursor.autowrap = 1;
+	cursor.bold = 0;
 }
 
 //endregion
@@ -156,6 +162,20 @@ screen_reset(void)
 	NOTIFY_DONE();
 }
 
+/**
+ * Reset the cursor
+ */
+void ICACHE_FLASH_ATTR
+screen_reset_cursor(void)
+{
+	NOTIFY_LOCK();
+	cursor.fg = termconf_scratch.default_fg;
+	cursor.bg = termconf_scratch.default_bg;
+	cursor.inverse = 0;
+	cursor.bold = 0;
+	NOTIFY_DONE();
+}
+
 /**
  * Clear screen area
  */
@@ -486,7 +506,7 @@ screen_inverse(bool inverse)
 void ICACHE_FLASH_ATTR
 screen_set_bright_fg(void)
 {
-	cursor.fg = (cursor.fg % 8) + 8;
+	cursor.fg = (Color) ((cursor.fg % 8) + 8);
 }
 
 //endregion
@@ -507,14 +527,14 @@ bool ICACHE_FLASH_ATTR screen_isCoordValid(int y, int x)
  * Set a character in the cursor color, move to right with wrap.
  */
 void ICACHE_FLASH_ATTR
-screen_putchar(char ch)
+screen_putchar(const char *ch)
 {
 	NOTIFY_LOCK();
 
 	Cell *c = &screen[cursor.x + cursor.y * W];
 
 	// Special treatment for CRLF
-	switch (ch) {
+	switch (ch[0]) {
 		case '\r':
 			screen_cursor_set_x(0);
 			goto done;
@@ -535,27 +555,34 @@ screen_putchar(char ch)
 			}
 			// erase target cell
 			c = &screen[cursor.x + cursor.y * W];
-			c->c = ' ';
+			c->c[0] = ' ';
+			c->c[1] = 0;
+			c->c[2] = 0;
+			c->c[3] = 0;
 			goto done;
 
 		case 9: // TAB
 			if (cursor.x<((W-1)-(W-1)%4)) {
-				c->c = ' ';
+				c->c[0] = ' ';
+				c->c[1] = 0;
+				c->c[2] = 0;
+				c->c[3] = 0;
 				do {
-					screen_putchar(' ');
+					screen_putchar(" ");
 				} while(cursor.x%4!=0);
 			}
 			goto done;
 
 		default:
-			if (ch < ' ') {
+			if (ch[0] < ' ') {
 				// Discard
 				warn("Ignoring control char %d", (int)ch);
 				goto done;
 			}
 	}
 
-	c->c = ch;
+	// copy unicode char
+	strncpy(c->c, ch, 4);
 
 	if (cursor.inverse) {
 		c->fg = cursor.bg;
@@ -627,10 +654,19 @@ void screen_dd(void)
 struct ScreenSerializeState {
 	Color lastFg;
 	Color lastBg;
-	char lastChar;
+	bool lastBold;
+	char lastChar[4];
 	int index;
 };
 
+void ICACHE_FLASH_ATTR
+encode2B(u16 number, WordB2 *stru)
+{
+	stru->lsb = (u8) (number % 127);
+	stru->msb = (u8) ((number - stru->lsb) / 127 + 1);
+	stru->lsb += 1;
+}
+
 /**
  * Serialize the screen to a data buffer. May need multiple calls if the buffer is insufficient in size.
  *
@@ -654,6 +690,7 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data)
 	}
 
 	Cell *cell, *cell0;
+	WordB2 w1, w2, w3, w4, w5;
 
 	size_t remain = buf_len; int used = 0;
 	char *bb = buffer;
@@ -669,27 +706,22 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data)
 		ss->index = 0;
 		ss->lastBg = 0;
 		ss->lastFg = 0;
-		ss->lastChar = '\0';
+		ss->lastBold = false;
-
+		memset(ss->lastChar, 0, 4); // this ensures the first char is never "repeat"
-		// TODO implement the new more efficient encoder!
+
-
+		encode2B((u16) H, &w1);
-		bufprint(
+		encode2B((u16) W, &w2);
-			"{"
+		encode2B((u16) cursor.y, &w3);
-			"\"w\":%d,"
+		encode2B((u16) cursor.x, &w4);
-			"\"h\":%d,"
+		encode2B((u16) (
-			"\"x\":%d,"
+			cursor.fg |
-			"\"y\":%d,"
+			(cursor.bg<<4) |
-			"\"fg\":%d,"
+			(cursor.bold?0x100:0) |
-			"\"bg\":%d,"
+			(cursor.visible?0x200:0))
-			"\"cv\":%d,"
+			, &w5);
-			"\"screen\":\"",
+
-			W,
+		// H W X Y Attribs
-			H,
+		bufprint("%c%c%c%c%c%c%c%c%c%c", w1.lsb, w1.msb, w2.lsb, w2.msb, w3.lsb, w3.msb, w4.lsb, w4.msb, w5.lsb, w5.msb);
-			cursor.x,
-			cursor.y,
-			cursor.fg,
-			cursor.bg,
-			cursor.visible);
 	}
 
 	int i = ss->index;
@@ -701,52 +733,60 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data)
 		while (i < W*H
 		       && cell->fg == ss->lastFg
 		       && cell->bg == ss->lastBg
-		       && cell->c == ss->lastChar) {
+		       && cell->bold == ss->lastBold
+		       && strneq(cell->c, ss->lastChar, 4)) {
 			// Repeat
 			repCnt++;
 			cell = &screen[++i];
 		}
 
 		if (repCnt == 0) {
+			// No repeat
+
 			// All this crap is needed because it's JSON and also
 			// embedded in HTML (hence the angle brackets)
 
-			if (cell0->fg == ss->lastFg && cell0->bg == ss->lastBg) {
+			// TODO use the encoding magic correctly
-				// same colors as previous
-				bufprint(",");
-			} else {
-				bufprint("%X%X", cell0->fg, cell0->bg);
-			}
 
-			char c = cell0->c;
+			if (cell0->bold != ss->lastBold || cell0->fg != ss->lastFg || cell0->bg != ss->lastBg) {
-			if (c == '"' || c == '\\') {
+				encode2B((u16) (
-				bufprint("\\%c", c);
+							 cell0->fg |
+							 (cell0->bg<<4) |
+							 (cell0->bold?0x100:0))
+					, &w1);
+				bufprint("\x01%c%c", w1.lsb, w1.msb);
 			}
-			else if (c == '<' || c == '>' || c == '\'' || c == '/' || c == '&') {
+
-				bufprint("\\u00%02X", (int)c);
+			// copy the symbol, until first 0 or reached 4 bytes
-			}
+			char c;
-			else {
+			int j = 0;
+			while ((c = cell->c[j++]) != 0 && j < 4) {
 				bufprint("%c", c);
 			}
 
+			// TODO do correctly JSON encoding
+//
+//			char c = cell0->c;
+//			if (c == '"' || c == '\\') {
+//				bufprint("\\%c", c);
+//			}
+//			else if (c == '<' || c == '>' || c == '\'' || c == '/' || c == '&') {
+//				bufprint("\\u00%02X", (int)c);
+//			}
+//			else {
+//				bufprint("%c", c);
+//			}
+
 			ss->lastFg = cell0->fg;
 			ss->lastBg = cell0->bg;
-			ss->lastChar = cell0->c;
+			ss->lastBold = cell0->bold;
+			memcpy(ss->lastChar, cell0->c, 4);
 
 			i++;
 		} else {
-			char code;
+			// Repeat count
-			if(repCnt<10) {
+			encode2B((u16) repCnt, &w1);
-				code = 'r';
+			bufprint("\x02%c%c", w1.lsb, w1.msb);
-			} else if(repCnt<100) {
-				code = 's';
-			} else if(repCnt<1000) {
-				code = 't';
-			} else {
-				code = 'u';
-			}
-
-			bufprint("%c%d", code, repCnt);
 		}
 	}
 
@@ -756,12 +796,7 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data)
 		return HTTPD_CGI_MORE;
 	}
 
-	if (remain >= 3) {
+	return HTTPD_CGI_DONE;
-		bufprint("\"\n}");
-		return HTTPD_CGI_DONE;
-	} else {
-		return HTTPD_CGI_MORE;
-	}
 }
 
 //endregion

user/screen.h

@@ -85,7 +85,7 @@ void terminal_apply_settings(void);
  * TODO May need adjusting if there are size problems when flashing the ESP.
  * We could also try to pack the Cell struct to a single 32bit word.
  */
-#define MAX_SCREEN_SIZE (80*25)
+#define MAX_SCREEN_SIZE (80*30)
 
 typedef enum {
 	CLEAR_TO_CURSOR=0, CLEAR_FROM_CURSOR=1, CLEAR_ALL=2
@@ -95,6 +95,14 @@ typedef uint8_t Color;
 
 httpd_cgi_state screenSerializeToBuffer(char *buffer, size_t buf_len, void **data);
 
+typedef struct {
+	u8 lsb;
+	u8 msb;
+} WordB2;
+
+/** Encode number to two nice ASCII bytes */
+void encode2B(u16 number, WordB2 *stru);
+
 /** Init the screen */
 void screen_init(void);
 
@@ -135,6 +143,9 @@ void screen_cursor_set_x(int x);
 /** Set cursor Y position */
 void screen_cursor_set_y(int y);
 
+/** Reset cursor attribs */
+void screen_reset_cursor(void);
+
 /** Relative cursor move */
 void screen_cursor_move(int dy, int dx);
 
@@ -168,8 +179,12 @@ void screen_set_colors(Color fg, Color bg);
 void screen_inverse(bool inverse);
 
 
-/** Set a character in the cursor color, move to right with wrap. */
+/**
-void screen_putchar(char c);
+ * Set a character in the cursor color, move to right with wrap.
+ * The character may be ASCII (then only one char is used), or
+ * unicode (then it can be 4 chars, or terminated by a zero)
+ */
+void screen_putchar(const char *ch);
 
 #if 0
 /** Debug dump */

user/serial.c

@@ -42,10 +42,5 @@ void ICACHE_FLASH_ATTR serialInit(void)
  */
 void ICACHE_FLASH_ATTR UART_HandleRxByte(char c)
 {
-	if (c > 0 && c < 127) {
+	ansi_parser(&c, 1);
-		// TODO buffering, do not run parser after just 1 char
-		ansi_parser(&c, 1);
-	} else {
-		warn("Bad char %d ('%c')", (unsigned char)c, c);
-	}
 }