From 1d6d6ee2280870b00e01fbb8342742ecf44ce4ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hru=C5=A1ka?= Date: Fri, 8 Sep 2017 23:53:23 +0200 Subject: [PATCH 01/62] Implement initial load through websocket to cut one unnecessary HTTP GET that caused conn pool overflow. Increased reload / retry delays to reduce server spamming by poorly loaded client. --- html_orig/jssrc/term_conn.js | 261 ++++++++++++++++++----------------- user/cgi_sockets.c | 15 +- 2 files changed, 146 insertions(+), 130 deletions(-) diff --git a/html_orig/jssrc/term_conn.js b/html_orig/jssrc/term_conn.js index 21be843..d9bbd57 100644 --- a/html_orig/jssrc/term_conn.js +++ b/html_orig/jssrc/term_conn.js @@ -1,131 +1,134 @@ /** Handle connections */ -var Conn = (function() { - var ws; - var heartbeatTout; - var pingIv; - var xoff = false; - var autoXoffTout; - - function onOpen(evt) { - console.log("CONNECTED"); - } - - function onClose(evt) { - console.warn("SOCKET CLOSED, code "+evt.code+". Reconnecting..."); - setTimeout(function() { - init(); - }, 200); - // this happens when the buffer gets fucked up via invalid unicode. - // we basically use polling instead of socket then - } - - function onMessage(evt) { - try { - // . = heartbeat - switch (evt.data.charAt(0)) { - case 'B': - case 'T': - case 'S': - Screen.load(evt.data); - break; - - case '-': - //console.log('xoff'); - xoff = true; - autoXoffTout = setTimeout(function(){xoff=false;}, 250); - break; - - case '+': - //console.log('xon'); - xoff = false; - clearTimeout(autoXoffTout); - break; - } - heartbeat(); - } catch(e) { - console.error(e); - } - } - - function canSend() { - return !xoff; - } - - function doSend(message) { - if (_demo) { - console.log("TX: ", message); - return true; // Simulate success - } - if (xoff) { - // TODO queue - console.log("Can't send, flood control."); - return false; - } - - if (!ws) return false; // for dry testing - if (ws.readyState != 1) { - console.error("Socket not ready"); - return false; - } - if (typeof message != "string") { - message = JSON.stringify(message); - } - ws.send(message); - return true; - } - - function init() { - if (_demo) { - console.log("Demo mode!"); - Screen.load(_demo_screen); - showPage(); - return; - } - heartbeat(); - - ws = new WebSocket("ws://"+_root+"/term/update.ws"); - ws.onopen = onOpen; - ws.onclose = onClose; - ws.onmessage = onMessage; - - console.log("Opening socket."); - - // Ask for initial data - $.get('http://'+_root+'/term/init', function(resp, status) { - if (status !== 200) location.reload(true); - console.log("Data received!"); - Screen.load(resp); - heartbeat(); - - showPage(); - }); - } - - function heartbeat() { - clearTimeout(heartbeatTout); - heartbeatTout = setTimeout(heartbeatFail, 2000); - } - - function heartbeatFail() { - console.error("Heartbeat lost, probing server..."); - pingIv = setInterval(function() { - console.log("> ping"); - $.get('http://'+_root+'/system/ping', function(resp, status) { - if (status == 200) { - clearInterval(pingIv); - console.info("Server ready, reloading page..."); - location.reload(); - } - }, { - timeout: 100, - }); - }, 500); - } - - return { - ws: null, - init: init, - send: doSend, - canSend: canSend, // check flood control - }; +var Conn = (function () { + var ws; + var heartbeatTout; + var pingIv; + var xoff = false; + var autoXoffTout; + var reconTout; + + var pageShown = false; + + function onOpen(evt) { + console.log("CONNECTED"); + doSend("i"); + } + + function onClose(evt) { + console.warn("SOCKET CLOSED, code " + evt.code + ". Reconnecting..."); + clearTimeout(reconTout); + reconTout = setTimeout(function () { + init(); + }, 2000); + // this happens when the buffer gets fucked up via invalid unicode. + // we basically use polling instead of socket then + } + + function onMessage(evt) { + try { + // . = heartbeat + switch (evt.data.charAt(0)) { + case 'B': + case 'T': + case 'S': + Screen.load(evt.data); + if(!pageShown) { + showPage(); + pageShown = true; + } + break; + + case '-': + //console.log('xoff'); + xoff = true; + autoXoffTout = setTimeout(function () { + xoff = false; + }, 250); + break; + + case '+': + //console.log('xon'); + xoff = false; + clearTimeout(autoXoffTout); + break; + } + heartbeat(); + } catch (e) { + console.error(e); + } + } + + function canSend() { + return !xoff; + } + + function doSend(message) { + if (_demo) { + console.log("TX: ", message); + return true; // Simulate success + } + if (xoff) { + // TODO queue + console.log("Can't send, flood control."); + return false; + } + + if (!ws) return false; // for dry testing + if (ws.readyState != 1) { + console.error("Socket not ready"); + return false; + } + if (typeof message != "string") { + message = JSON.stringify(message); + } + ws.send(message); + return true; + } + + function init() { + if (_demo) { + console.log("Demo mode!"); + Screen.load(_demo_screen); + showPage(); + return; + } + + clearTimeout(reconTout); + clearTimeout(heartbeatTout); + + ws = new WebSocket("ws://" + _root + "/term/update.ws"); + ws.onopen = onOpen; + ws.onclose = onClose; + ws.onmessage = onMessage; + console.log("Opening socket."); + heartbeat(); + } + + function heartbeat() { + clearTimeout(heartbeatTout); + heartbeatTout = setTimeout(heartbeatFail, 2000); + } + + function heartbeatFail() { + console.error("Heartbeat lost, probing server..."); + pingIv = setInterval(function () { + console.log("> ping"); + $.get('http://' + _root + '/system/ping', function (resp, status) { + if (status == 200) { + clearInterval(pingIv); + console.info("Server ready, reloading page..."); + location.reload(); + } + }, { + timeout: 100, + }); + }, 1000); + } + + return { + ws: null, + init: init, + send: doSend, + canSend: canSend, // check flood control + }; })(); diff --git a/user/cgi_sockets.c b/user/cgi_sockets.c index 2e6d8bc..e409c8f 100644 --- a/user/cgi_sockets.c +++ b/user/cgi_sockets.c @@ -47,6 +47,7 @@ notifyCooldownTimCb(void *arg) static void ICACHE_FLASH_ATTR notifyContentTimCb(void *arg) { + Websock *ws = arg; void *data = NULL; int max_bl, total_bl; char sock_buff[SOCK_BUF_LEN]; @@ -65,8 +66,14 @@ notifyContentTimCb(void *arg) int flg = 0; if (cont == HTTPD_CGI_MORE) flg |= WEBSOCK_FLAG_MORE; if (i > 0) flg |= WEBSOCK_FLAG_CONT; - cgiWebsockBroadcast(URL_WS_UPDATE, sock_buff, (int) strlen(sock_buff), flg); + if (ws) { + cgiWebsocketSend(ws, sock_buff, (int) strlen(sock_buff), flg); + } else { + cgiWebsockBroadcast(URL_WS_UPDATE, sock_buff, (int) strlen(sock_buff), flg); + } if (cont == HTTPD_CGI_DONE) break; + + system_soft_wdt_feed(); } // cleanup @@ -251,6 +258,12 @@ void ICACHE_FLASH_ATTR updateSockRx(Websock *ws, char *data, int len, int flags) } break; + case 'i': + // requests initial load + dbg("Client requests initial load"); + notifyContentTimCb(ws); + break; + case 'm': case 'p': case 'r': From 7d6cb2b660aa0d7b4a77a69953e670da1845722a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hru=C5=A1ka?= Date: Sat, 9 Sep 2017 01:57:15 +0200 Subject: [PATCH 02/62] screen serialize speedup + fix bad utf encode --- user/screen.c | 85 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/user/screen.c b/user/screen.c index 4ac8e37..45763e2 100644 --- a/user/screen.c +++ b/user/screen.c @@ -7,6 +7,7 @@ #include "apars_logging.h" #include "jstring.h" #include "character_sets.h" +#include "uart_driver.h" TerminalConfigBundle * const termconf = &persist.current.termconf; TerminalConfigBundle termconf_scratch; @@ -1282,10 +1283,10 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data) } Cell *cell, *cell0; - WordB2 w1, w2, w3, w4, w5; + WordB2 w1; WordB3 lw1; - size_t remain = buf_len; int used = 0; + size_t remain = buf_len; char *bb = buffer; // Ideally we'd use snprintf here! @@ -1294,6 +1295,34 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data) if(used>0) { bb += used; remain -= used; } \ } while(0) +#define bufput_c(c) do { \ + *bb = (char)c; bb++; \ + remain--; \ + } while(0) + +#define bufput_2B(n) do { \ + encode2B((u16) n, &w1); \ + bufput_c(w1.lsb); \ + bufput_c(w1.msb); \ + } while(0) + +#define bufput_3B(n) do { \ + encode3B((u16) n, &lw1); \ + bufput_c(lw1.lsb); \ + bufput_c(lw1.msb); \ + bufput_c(lw1.xsb); \ + } while(0) + +#define bufput_t2B(t, n) do { \ + bufput_c(t); \ + bufput_2B(n); \ + } while(0) + +#define bufput_t3B(t, n) do { \ + bufput_c(t); \ + bufput_3B(n); \ + } while(0) + if (ss == NULL) { *data = ss = malloc(sizeof(struct ScreenSerializeState)); ss->index = 0; @@ -1302,25 +1331,23 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data) ss->lastAttrs = 0; memset(ss->lastChar, 0, 4); // this ensures the first char is never "repeat" - encode2B((u16) H, &w1); - encode2B((u16) W, &w2); - encode2B((u16) cursor.y, &w3); - encode2B((u16) cursor.x, &w4); - encode2B((u16) ( - (scr.cursor_visible ? 1<<0 : 0) | - (cursor.hanging ? 1<<1 : 0) | - (scr.cursors_alt_mode ? 1<<2 : 0) | - (scr.numpad_alt_mode ? 1<<3 : 0) | - (termconf->fn_alt_mode ? 1<<4 : 0) | - ((mouse_tracking.mode>MTM_NONE) ? 1<<5 : 0) | // disables context menu - ((mouse_tracking.mode>=MTM_NORMAL) ? 1<<6 : 0) | // disables selecting - (termconf_scratch.show_buttons ? 1<<7 : 0) | - (termconf_scratch.show_config_links ? 1<<8 : 0) - ) - , &w5); - + bufput_c('S'); // H W X Y Attribs - bufprint("S%c%c%c%c%c%c%c%c%c%c", w1.lsb, w1.msb, w2.lsb, w2.msb, w3.lsb, w3.msb, w4.lsb, w4.msb, w5.lsb, w5.msb); + bufput_2B(H); + bufput_2B(W); + bufput_2B(cursor.y); + bufput_2B(cursor.x); + bufput_2B(( + (scr.cursor_visible ? 1<<0 : 0) | + (cursor.hanging ? 1<<1 : 0) | + (scr.cursors_alt_mode ? 1<<2 : 0) | + (scr.numpad_alt_mode ? 1<<3 : 0) | + (termconf->fn_alt_mode ? 1<<4 : 0) | + ((mouse_tracking.mode>MTM_NONE) ? 1<<5 : 0) | // disables context menu + ((mouse_tracking.mode>=MTM_NORMAL) ? 1<<6 : 0) | // disables selecting + (termconf_scratch.show_buttons ? 1<<7 : 0) | + (termconf_scratch.show_config_links ? 1<<8 : 0) + )); } int i = ss->index; @@ -1356,25 +1383,23 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data) } if (!changeAttrs && changeColors) { - encode2B(fg | (bg<<4), &w1); - bufprint("\x03%c%c", w1.lsb, w1.msb); + bufput_t2B('\x03', fg | (bg<<4)); } else if (changeAttrs && !changeColors) { // attrs only - encode2B(cell0->attrs, &w1); - bufprint("\x04%c%c", w1.lsb, w1.msb); + bufput_t2B('\x04', cell0->attrs); } else if (changeAttrs && changeColors) { // colors and attrs - encode3B((u32) (fg | (bg<<4) | (cell0->attrs<<8)), &lw1); - bufprint("\x01%c%c%c", lw1.lsb, lw1.msb, lw1.xsb); + bufput_t3B('\x01', (u32) (fg | (bg<<4) | (cell0->attrs<<8))); } // copy the symbol, until first 0 or reached 4 bytes char c; int j = 0; - while ((c = cell->c[j++]) != 0 && j < 4) { - bufprint("%c", c); + while ((c = cell->c[j]) != 0 && j < 4) { + bufput_c(c); + j++; } ss->lastFg = cell0->fg; @@ -1385,12 +1410,12 @@ screenSerializeToBuffer(char *buffer, size_t buf_len, void **data) i++; } else { // Repeat count - encode2B((u16) repCnt, &w1); - bufprint("\x02%c%c", w1.lsb, w1.msb); + bufput_t2B('\x02', repCnt); } } ss->index = i; + bufput_c('\0'); // terminate the string if (i < W*H-1) { return HTTPD_CGI_MORE; From 830ec073bfd800846cf681749b0aaad136ea0af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Hru=C5=A1ka?= Date: Sat, 9 Sep 2017 03:03:12 +0200 Subject: [PATCH 03/62] fix #109 --- .gitignore | 1 + html_orig/pages/term.php | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6702851..675c98f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ espfs/espfstest/espfstest html_compressed/ ldscript_memspecific.ld eagle.app.sym +node_modules # Garbage added by CLion .idea/ diff --git a/html_orig/pages/term.php b/html_orig/pages/term.php index ed9e627..ea84d53 100644 --- a/html_orig/pages/term.php +++ b/html_orig/pages/term.php @@ -56,7 +56,7 @@