Merge branch 'utf-removal-fix' into work

esphttpdconfig.mk

@@ -48,7 +48,7 @@ GLOBAL_CFLAGS = \
     -DDEBUG_CGI=0 \
     -DDEBUG_WIFI=0 \
     -DDEBUG_WS=0 \
-    -DDEBUG_ANSI=1 \
+    -DDEBUG_ANSI=0 \
     -DDEBUG_ANSI_NOIMPL=1 \
     -DDEBUG_INPUT=0 \
     -DDEBUG_HEAP=1 \

front-end

@@ -1 +1 @@
-Subproject commit 6c6424877c49e3e23f563067a78e79338226359d
+Subproject commit c68017bd4ad387786dd17b213be9bdd29fe4c0ea

user/cgi_sockets.c

@@ -35,6 +35,8 @@ volatile int active_clients = 0;
 // this might glitch, very rarely.
 // it's recommended to put some delay between setting labels and updating the screen.
 
+static void resetHeartbeatTimer(void);
+
 /**
  * Cooldown delay is over
  * @param arg
@@ -61,7 +63,8 @@ notifyContentTimCb(void *arg)
 
 	if (!notify_available || notify_cooldown || (max_bl > 2048)) { // do not send if we have anything significant backlogged
 		// postpone a little
-		TIMER_START(&notifyContentTim, notifyContentTimCb, 5, 0);
+		TIMER_START(&notifyContentTim, notifyContentTimCb, 4, 0);
+		inp_dbg("postpone notify content");
 		return;
 	}
 	notify_available = false;
@@ -75,6 +78,7 @@ notifyContentTimCb(void *arg)
 			cgiWebsocketSend(ws, sock_buff, (int) strlen(sock_buff), flg);
 		} else {
 			cgiWebsockBroadcast(URL_WS_UPDATE, sock_buff, (int) strlen(sock_buff), flg);
+			resetHeartbeatTimer();
 		}
 		if (cont == HTTPD_CGI_DONE) break;
 
@@ -97,17 +101,25 @@ notifyContentTimCb(void *arg)
 static void ICACHE_FLASH_ATTR
 notifyLabelsTimCb(void *arg)
 {
+	Websock *ws = arg;
 	char sock_buff[SOCK_BUF_LEN];
 
 	if (!notify_available || notify_cooldown) {
 		// postpone a little
-		TIMER_START(&notifyLabelsTim, notifyLabelsTimCb, 1, 0);
+		TIMER_START(&notifyLabelsTim, notifyLabelsTimCb, 7, 0);
+		inp_dbg("postpone notify labels");
 		return;
 	}
 	notify_available = false;
 
 	screenSerializeLabelsToBuffer(sock_buff, SOCK_BUF_LEN);
-	cgiWebsockBroadcast(URL_WS_UPDATE, sock_buff, (int) strlen(sock_buff), 0);
+
+	if (ws) {
+		cgiWebsocketSend(ws, sock_buff, (int) strlen(sock_buff), 0);
+	} else {
+		cgiWebsockBroadcast(URL_WS_UPDATE, sock_buff, (int) strlen(sock_buff), 0);
+		resetHeartbeatTimer();
+	}
 
 	notify_cooldown = true;
 	notify_available = true;
@@ -123,6 +135,7 @@ send_beep(void)
 
 	// here's some potential for a race error with the other broadcast functions :C
 	cgiWebsockBroadcast(URL_WS_UPDATE, "B", 1, 0);
+	resetHeartbeatTimer();
 }
 
 
@@ -135,6 +148,7 @@ notify_growl(char *msg)
 	// TODO via timer...
 	// here's some potential for a race error with the other broadcast functions :C
 	cgiWebsockBroadcast(URL_WS_UPDATE, msg, (int) strlen(msg), 0);
+	resetHeartbeatTimer();
 }
 
 
@@ -148,13 +162,14 @@ void ICACHE_FLASH_ATTR screen_notifyChange(ScreenNotifyChangeTopic topic)
 	if (active_clients == 0) return;
 
 	// this is probably not needed here - ensure timeout is not 0
-	if (termconf->display_tout_ms == 0) termconf->display_tout_ms = SCR_DEF_DISPLAY_TOUT_MS;
+	if (termconf->display_tout_ms == 0)
+		termconf->display_tout_ms = SCR_DEF_DISPLAY_TOUT_MS;
 
 	// NOTE: the timers are restarted if already running
 
 	if (topic == CHANGE_LABELS) {
 		// separate timer from content change timer, to avoid losing that update
-		TIMER_START(&notifyLabelsTim, notifyLabelsTimCb, termconf->display_tout_ms, 0);
+		TIMER_START(&notifyLabelsTim, notifyLabelsTimCb, termconf->display_tout_ms+2, 0); // this delay is useful when both are fired at once on screen reset
 	}
 	else if (topic == CHANGE_CONTENT) {
 		// throttle delay
@@ -170,7 +185,7 @@ void ICACHE_FLASH_ATTR screen_notifyChange(ScreenNotifyChangeTopic topic)
  * @param button - which button, 1-based. 0=none
  * @param mods - modifier keys bitmap: meta|alt|shift|ctrl
  */
-void ICACHE_FLASH_ATTR sendMouseAction(char evt, int y, int x, int button, u8 mods)
+static void ICACHE_FLASH_ATTR sendMouseAction(char evt, int y, int x, int button, u8 mods)
 {
 	// one-based
 	x++;
@@ -239,7 +254,7 @@ void ICACHE_FLASH_ATTR sendMouseAction(char evt, int y, int x, int button, u8 mo
 }
 
 /** Socket received a message */
-void ICACHE_FLASH_ATTR updateSockRx(Websock *ws, char *data, int len, int flags)
+static void ICACHE_FLASH_ATTR updateSockRx(Websock *ws, char *data, int len, int flags)
 {
 	// Add terminator if missing (seems to randomly happen)
 	data[len] = 0;
@@ -281,7 +296,8 @@ void ICACHE_FLASH_ATTR updateSockRx(Websock *ws, char *data, int len, int flags)
 		case 'i':
 			// requests initial load
 			inp_dbg("Client requests initial load");
-			notifyContentTimCb(ws);
+			notifyContentTimCb(ws); // delay??
+			notifyLabelsTimCb(ws);
 			break;
 
 		case 'm':
@@ -304,16 +320,33 @@ void ICACHE_FLASH_ATTR updateSockRx(Websock *ws, char *data, int len, int flags)
 }
 
 /** Send a heartbeat msg */
-void ICACHE_FLASH_ATTR heartbeatTimCb(void *unused)
+static void ICACHE_FLASH_ATTR heartbeatTimCb(void *unused)
 {
-	if (notify_available && active_clients > 0) {
+	if (active_clients > 0) {
-		// Heartbeat packet - indicate we're still connected
+		if (notify_available) {
-		// JS reloads the page if heartbeat is lost for a couple seconds
+			inp_dbg(".");
-		cgiWebsockBroadcast(URL_WS_UPDATE, ".", 1, 0);
+
+			// Heartbeat packet - indicate we're still connected
+			// JS reloads the page if heartbeat is lost for a couple seconds
+			cgiWebsockBroadcast(URL_WS_UPDATE, ".", 1, 0);
+
+			// schedule next tick
+			TIMER_START(&heartbeatTim, heartbeatTimCb, HB_TIME, 0);
+		} else {
+			// postpone...
+			TIMER_START(&heartbeatTim, heartbeatTimCb, 10, 0);
+			inp_dbg("postpone heartbeat");
+		}
 	}
 }
 
-void ICACHE_FLASH_ATTR closeSockCb(Websock *ws)
+static void ICACHE_FLASH_ATTR resetHeartbeatTimer(void)
+{
+	TIMER_START(&heartbeatTim, heartbeatTimCb, HB_TIME, 0);
+}
+
+
+static void ICACHE_FLASH_ATTR closeSockCb(Websock *ws)
 {
 	active_clients--;
 	if (active_clients <= 0) {
@@ -340,7 +373,7 @@ void ICACHE_FLASH_ATTR updateSockConnect(Websock *ws)
 			UART_SendAsync("\x1b[I", 3);
 		}
 
-		TIMER_START(&heartbeatTim, heartbeatTimCb, HB_TIME, 1);
+		resetHeartbeatTimer();
 	}
 
 	active_clients++;
@@ -348,14 +381,14 @@ void ICACHE_FLASH_ATTR updateSockConnect(Websock *ws)
 
 ETSTimer xonTim;
 
-void ICACHE_FLASH_ATTR notify_empty_txbuf_cb(void *unused)
+static void ICACHE_FLASH_ATTR notify_empty_txbuf_cb(void *unused)
 {
 	UART_WriteChar(UART1, '+', 100);
 	cgiWebsockBroadcast(URL_WS_UPDATE, "+", 1, 0);
+	resetHeartbeatTimer();
 	browser_wants_xon = false;
 }
 
-
 void notify_empty_txbuf(void)
 {
 	if (browser_wants_xon) {

user/cgi_term_cfg.c

@@ -43,10 +43,10 @@ cgiTermCfgSetParams(HttpdConnData *connData)
 
 	// width and height must always go together so we can do max size validation
 	if (GET_ARG("term_width")) {
-		cgi_dbg("Default screen width: %s", buff);
-		w = atoi(buff);
 		do {
-			if (w <= 1) {
+			cgi_dbg("Default screen width: %s", buff);
+			w = atoi(buff);
+			if (w < 1) {
 				cgi_warn("Bad width: \"%s\"", buff);
 				redir_url += sprintf(redir_url, "term_width,");
 				break;
@@ -61,7 +61,7 @@ cgiTermCfgSetParams(HttpdConnData *connData)
 
 			cgi_dbg("Default screen height: %s", buff);
 			h = atoi(buff);
-			if (h <= 1) {
+			if (h < 1) {
 				cgi_warn("Bad height: \"%s\"", buff);
 				redir_url += sprintf(redir_url, "term_height,");
 				break;

user/screen.c

@@ -60,8 +60,8 @@ static struct {
 	char last_char[4];
 } scr;
 
-#define R0 scr.vm0
+#define TOP scr.vm0
-#define R1 scr.vm1
+#define BTM scr.vm1
 #define RH (scr.vm1 - scr.vm0 + 1)
 // horizontal edges - will be useful if horizontal margin is implemented
 //#define C0 0
@@ -154,7 +154,7 @@ static volatile int notifyLock = 0;
 									if (cursor.hanging && cursor.x != W-1) cursor.hanging = false; \
 								} while(false)
 
-#define cursor_inside_region() (cursor.y >= R0 && cursor.y <= R1)
+#define cursor_inside_region() (cursor.y >= TOP && cursor.y <= BTM)
 
 //region --- Settings ---
 
@@ -248,18 +248,7 @@ screen_init(void)
 {
 	if(DEBUG_HEAP) dbg("Screen buffer size = %d bytes", sizeof(screen));
 
-	NOTIFY_LOCK();
-	Cell sample;
-	sample.symbol = ' ';
-	sample.fg = 0;
-	sample.bg = 0;
-	sample.attrs = 0; // use default colors
-	for (unsigned int i = 0; i < MAX_SCREEN_SIZE; i++) {
-		memcpy(&screen[i], &sample, sizeof(Cell));
-	}
-
 	screen_reset();
-	NOTIFY_DONE();
 }
 
 /**
@@ -288,16 +277,13 @@ screen_reset_on_resize(void)
 	ansi_dbg("Screen partial reset due to resize");
 	NOTIFY_LOCK();
 
-	cursor.x = 0;
+	cursor_reset();
-	cursor.y = 0;
-	cursor.hanging = false;
 
 	scr.vm0 = 0;
 	scr.vm1 = H-1;
 
 	// size is left unchanged
-	screen_clear(CLEAR_ALL);
+	screen_clear(CLEAR_ALL); // also clears utf cache
-	unicode_cache_clear();
 
 	NOTIFY_DONE();
 }
@@ -314,18 +300,11 @@ screen_reset_sgr(void)
 	cursor.conceal = false;
 }
 
-/**
+static void ICACHE_FLASH_ATTR
- * Reset the screen - called by ESC c
+screen_reset_do(bool size, bool labels)
- */
-void ICACHE_FLASH_ATTR
-screen_reset(void)
 {
-	ansi_dbg("Screen reset.");
 	NOTIFY_LOCK();
 
-	cursor_reset();
-	unicode_cache_clear();
-
 	// DECopts
 	scr.cursor_visible = true;
 	scr.insert_mode = false;
@@ -339,17 +318,21 @@ screen_reset(void)
 	termconf_live.crlf_mode = termconf->crlf_mode;
 	scr.reverse_video = false;
 
-	scr.vm0 = 0;
-	scr.vm1 = H-1;
-
 	state_backup.alternate_active = false;
 
 	mouse_tracking.encoding = MTE_SIMPLE;
 	mouse_tracking.focus_tracking = false;
 	mouse_tracking.mode = MTM_NONE;
 
-	// size is left unchanged
+	if (size) {
-	screen_clear(CLEAR_ALL);
+		W = termconf->width;
+		H = termconf->height;
+	}
+
+	scr.vm0 = 0;
+	scr.vm1 = H - 1;
+	cursor_reset();
+	screen_clear(CLEAR_ALL); // also clears utf cache
 
 	// Set initial tabstops
 	for (int i = 0; i < TABSTOP_WORDS; i++) {
@@ -372,8 +355,30 @@ screen_reset(void)
 	opt_backup.show_config_links = termconf_live.show_config_links;
 
 	NOTIFY_DONE();
+
+	if (labels) {
+		strcpy(termconf_live.title, termconf->title);
+
+		for (int i = 1; i <= TERM_BTN_COUNT; i++) {
+			strcpy(termconf_live.btn[i], termconf->btn[i]);
+			strcpy(termconf_live.btn_msg[i], termconf->btn_msg[i]);
+		}
+
+		screen_notifyChange(CHANGE_LABELS);
+	}
 }
 
+/**
+ * Reset the screen - called by ESC c
+ */
+void ICACHE_FLASH_ATTR
+screen_reset(void)
+{
+	ansi_dbg("Screen reset.");
+	screen_reset_do(true, true);
+}
+
+
 /**
  * Swap screen buffer / state
  * this is CSI ? 47/1047/1049 h/l
@@ -545,9 +550,13 @@ screen_tab_reverse(int count)
 
 /**
  * Clear range, inclusive
+ *
+ * @param from - starting absolute position
+ * @param to - ending absolute position
+ * @param clear_utf - release any encountered utf8
  */
 static void ICACHE_FLASH_ATTR
-clear_range(unsigned int from, unsigned int to)
+clear_range_do(unsigned int from, unsigned int to, bool clear_utf)
 {
 	if (to >= W*H) to = W*H-1;
 
@@ -559,12 +568,145 @@ clear_range(unsigned int from, unsigned int to)
 	sample.attrs = (CellAttrs) (cursor.attrs & (ATTR_FG | ATTR_BG));
 
 	for (unsigned int i = from; i <= to; i++) {
-		UnicodeCacheRef symbol = screen[i].symbol;
+		if (clear_utf) {
-		if (IS_UNICODE_CACHE_REF(symbol)) unicode_cache_remove(symbol);
+			UnicodeCacheRef symbol = screen[i].symbol;
+			if (IS_UNICODE_CACHE_REF(symbol)) unicode_cache_remove(symbol);
+		}
 		memcpy(&screen[i], &sample, sizeof(Cell));
 	}
 }
 
+/**
+ * Clear range, inclusive, freeing any encountered UTF8 from the cache
+ *
+ * @param from - starting absolute position
+ * @param to - ending absolute position
+ */
+static inline void ICACHE_FLASH_ATTR
+clear_range_utf(unsigned int from, unsigned int to)
+{
+	clear_range_do(from, to, true);
+}
+
+/**
+ * Clear range, inclusive. Do not release utf characters
+ *
+ * @param from - starting absolute position
+ * @param to - ending absolute position
+ */
+static inline void ICACHE_FLASH_ATTR
+clear_range_noutf(unsigned int from, unsigned int to)
+{
+	clear_range_do(from, to, false);
+}
+
+/**
+ * Free a utf8 reference character in a cell
+ *
+ * @param row
+ * @param col
+ */
+static inline void ICACHE_FLASH_ATTR
+utf_free_cell(int row, int col)
+{
+	//dbg("free cell (row %d) %d", row, col);
+	UnicodeCacheRef symbol = screen[row * W + col].symbol;
+	if (IS_UNICODE_CACHE_REF(symbol))
+		unicode_cache_remove(symbol);
+}
+
+/**
+ * Back-up utf8 reference in a cell (i.e. increment the counter,
+ * so 1 subsequent free has no effect)
+ *
+ * @param row
+ * @param col
+ */
+static inline void ICACHE_FLASH_ATTR
+utf_backup_cell(int row, int col)
+{
+	//dbg("backup cell (row %d) %d", row, col);
+	UnicodeCacheRef symbol = screen[row * W + col].symbol;
+	if (IS_UNICODE_CACHE_REF(symbol))
+		unicode_cache_inc(symbol);
+}
+
+/**
+ * Duplicate a cell within a row
+ * @param row - row to work on
+ * @param dest_col - destination column
+ * @param src_col - source column
+ */
+static inline void ICACHE_FLASH_ATTR
+copy_cell(int row, int dest_col, int src_col)
+{
+	//dbg("copy cell (row %d) %d -> %d", row, src_col, dest_col);
+	memcpy(screen+row*W+dest_col, screen+row*W+src_col, sizeof(Cell));
+}
+
+/**
+ * Free all utf8 on a row
+ *
+ * @param row
+ */
+static inline void ICACHE_FLASH_ATTR
+utf_free_row(int row)
+{
+	//dbg("free row %d", row);
+	for (int col = 0; col < W; col++) {
+		utf_free_cell(row, col);
+	}
+}
+
+/**
+ * Back-up all utf8 refs on a row
+ *
+ * @param row
+ */
+static inline void ICACHE_FLASH_ATTR
+utf_backup_row(int row)
+{
+	//dbg("backup row %d", row);
+	for (int col = 0; col < W; col++) {
+		utf_backup_cell(row, col);
+	}
+}
+
+/**
+ * Duplicate a row
+ *
+ * @param dest - destination row number (0-based)
+ * @param src  - source row number (0-based)
+ */
+static inline void ICACHE_FLASH_ATTR
+copy_row(int dest, int src)
+{
+	//dbg("copy row %d -> %d", src, dest);
+	memcpy(screen + dest * W, screen + src * W, sizeof(Cell) * W);
+}
+
+/**
+ * Clear a row, do nothing with the utf8 cache
+ *
+ * @param row
+ */
+static inline void ICACHE_FLASH_ATTR
+clear_row_noutf(int row)
+{
+	clear_range_noutf(row * W, (row + 1) * W - 1);
+}
+
+/**
+ * Clear a row, freeing any utf8 refs
+ *
+ * @param row
+ */
+static inline void ICACHE_FLASH_ATTR
+clear_row_utf(int row)
+{
+	clear_range_utf(row * W, (row + 1) * W - 1);
+}
+
 /**
  * Clear screen area
  */
@@ -575,15 +717,17 @@ screen_clear(ClearMode mode)
 	NOTIFY_LOCK();
 	switch (mode) {
 		case CLEAR_ALL:
-			clear_range(0, W * H - 1);
+			unicode_cache_clear();
+			clear_range_noutf(0, W * H - 1);
+			scr.last_char[0]  = 0;
 			break;
 
 		case CLEAR_FROM_CURSOR:
-			clear_range((cursor.y * W) + cursor.x, W * H - 1);
+			clear_range_utf((cursor.y * W) + cursor.x, W * H - 1);
 			break;
 
 		case CLEAR_TO_CURSOR:
-			clear_range(0, (cursor.y * W) + cursor.x);
+			clear_range_utf(0, (cursor.y * W) + cursor.x);
 			break;
 	}
 	NOTIFY_DONE();
@@ -598,15 +742,15 @@ screen_clear_line(ClearMode mode)
 	NOTIFY_LOCK();
 	switch (mode) {
 		case CLEAR_ALL:
-			clear_range(cursor.y * W, (cursor.y + 1) * W - 1);
+			clear_row_utf(cursor.y);
 			break;
 
 		case CLEAR_FROM_CURSOR:
-			clear_range(cursor.y * W + cursor.x, (cursor.y + 1) * W - 1);
+			clear_range_utf(cursor.y * W + cursor.x, (cursor.y + 1) * W - 1);
 			break;
 
 		case CLEAR_TO_CURSOR:
-			clear_range(cursor.y * W, cursor.y * W + cursor.x);
+			clear_range_utf(cursor.y * W, cursor.y * W + cursor.x);
 			break;
 	}
 	NOTIFY_DONE();
@@ -615,14 +759,13 @@ screen_clear_line(ClearMode mode)
 void ICACHE_FLASH_ATTR
 screen_clear_in_line(unsigned int count)
 {
+	NOTIFY_LOCK();
 	if (cursor.x + count >= W) {
 		screen_clear_line(CLEAR_FROM_CURSOR);
+	} else {
+		clear_range_utf(cursor.y * W + cursor.x, cursor.y * W + cursor.x + count - 1);
 	}
-	else {
+	NOTIFY_DONE();
-		NOTIFY_LOCK();
-		clear_range(cursor.y * W + cursor.x, cursor.y * W + cursor.x + count - 1);
-		NOTIFY_DONE();
-	}
 }
 
 void ICACHE_FLASH_ATTR
@@ -631,24 +774,24 @@ screen_insert_lines(unsigned int lines)
 	if (!cursor_inside_region()) return; // can't insert if not inside region
 	NOTIFY_LOCK();
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
-
 	// shift the following lines
 	int targetStart = cursor.y + lines;
-	if (targetStart > R1) {
+	if (targetStart > BTM) {
-		targetStart = R1-1;
+		clear_range_utf(cursor.y*W, (BTM+1)*W-1);
 	} else {
 		// do the moving
-		for (int i = R1; i >= targetStart; i--) {
+		for (int i = BTM; i >= targetStart; i--) {
-			memcpy(screen+i*W, screen+(i-lines)*W, sizeof(Cell)*W);
+			utf_free_row(i); // release old characters
+			copy_row(i, i - lines);
+			if (i != targetStart) utf_backup_row(i);
 		}
-	}
 
-	// clear the first line
+		// clear the first line
-	screen_clear_line(CLEAR_ALL);
+		clear_row_noutf(cursor.y);
-	// copy it to the rest of the cleared region
+		// copy it to the rest of the cleared region
-	for (int i = cursor.y+1; i < targetStart; i++) {
+		for (int i = cursor.y+1; i < targetStart; i++) {
-		memcpy(screen+i*W, screen+cursor.y*W, sizeof(Cell)*W);
+			copy_row(i, cursor.y);
+		}
 	}
 	NOTIFY_DONE();
 }
@@ -659,21 +802,22 @@ screen_delete_lines(unsigned int lines)
 	if (!cursor_inside_region()) return; // can't delete if not inside region
 	NOTIFY_LOCK();
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
-
 	// shift lines up
-	int targetEnd = R1 - lines - 1;
+	int movedBlockEnd = BTM - lines ;
-	if (targetEnd <= cursor.y) {
+	if (movedBlockEnd <= cursor.y) {
-		targetEnd = cursor.y;
+		// clear the entire rest of the screen
+		movedBlockEnd = cursor.y;
+		clear_range_utf(movedBlockEnd*W, (BTM+1)*W-1);
 	} else {
-		// do the moving
+		// move some lines up, clear the rest
-		for (int i = cursor.y; i <= targetEnd; i++) {
+		for (int i = cursor.y; i <= movedBlockEnd; i++) {
-			memcpy(screen+i*W, screen+(i+lines)*W, sizeof(Cell)*W);
+			utf_free_row(i);
+			copy_row(i, i+lines);
+			if (i != movedBlockEnd) utf_backup_row(i);
 		}
+		clear_range_noutf((movedBlockEnd+1)*W, (BTM+1)*W-1);
 	}
 
-	// clear the rest
-	clear_range(targetEnd*W, W*R1);
 	NOTIFY_DONE();
 }
 
@@ -682,19 +826,21 @@ screen_insert_characters(unsigned int count)
 {
 	NOTIFY_LOCK();
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
+	// shove rest of the line to the right
 
 	int targetStart = cursor.x + count;
 	if (targetStart >= W) {
-		targetStart = W-1;
+		// all rest of line was cleared
+		clear_range_utf(cursor.y * W + cursor.x, (cursor.y + 1) * W - 1);
 	} else {
 		// do the moving
 		for (int i = W-1; i >= targetStart; i--) {
-			memcpy(screen+cursor.y*W+i, screen+cursor.y*W+(i-count), sizeof(Cell));
+			utf_free_cell(cursor.y, i);
+			copy_cell(cursor.y, i, i - count);
+			utf_backup_cell(cursor.y, i);
 		}
+		clear_range_utf(cursor.y * W + cursor.x, cursor.y * W + targetStart - 1);
 	}
-
-	clear_range(cursor.y*W+cursor.x, cursor.y*W+targetStart-1);
 	NOTIFY_DONE();
 }
 
@@ -703,21 +849,24 @@ screen_delete_characters(unsigned int count)
 {
 	NOTIFY_LOCK();
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
+	// pull rest of the line to the left
-	int targetEnd = W - count;
-	if (targetEnd > cursor.x) {
-		// do the moving
-		for (int i = cursor.x; i <= targetEnd; i++) {
-			memcpy(screen+cursor.y*W+i, screen+cursor.y*W+(i+count), sizeof(Cell));
-		}
-	}
 
-	if (targetEnd <= cursor.x) {
+	int movedBlockEnd = W - count;
+	if (movedBlockEnd > cursor.x) {
+		// partial line delete / move
+
+		for (int i = cursor.x; i <= movedBlockEnd; i++) {
+			utf_free_cell(cursor.y, i);
+			copy_cell(cursor.y, i, i + count);
+			utf_backup_cell(cursor.y, i);
+		}
+		// clear original positions of the moved characters
+		clear_range_noutf(cursor.y * W + (W - count), (cursor.y + 1) * W - 1);
+	} else {
+		// all rest was cleared
 		screen_clear_line(CLEAR_FROM_CURSOR);
 	}
-	else {
+
-		clear_range(cursor.y * W + (W - count), (cursor.y + 1) * W - 1);
-	}
 	NOTIFY_DONE();
 }
 //endregion
@@ -728,7 +877,7 @@ void ICACHE_FLASH_ATTR
 screen_fill_with_E(void)
 {
 	NOTIFY_LOCK();
-	screen_reset(); // based on observation from xterm
+	screen_reset_do(false, false); // based on observation from xterm
 
 	Cell sample;
 	sample.symbol = 'E';
@@ -798,7 +947,8 @@ screen_scroll_up(unsigned int lines)
 {
 	NOTIFY_LOCK();
 	if (lines >= RH) {
-		clear_range(R0*W, (R1+1)*W-1);
+		// clear entire region
+		clear_range_utf(TOP * W, (BTM + 1) * W - 1);
 		goto done;
 	}
 
@@ -807,16 +957,16 @@ screen_scroll_up(unsigned int lines)
 		goto done;
 	}
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
-
 	int y;
-	for (y = R0; y <= R1 - lines; y++) {
+	for (y = TOP; y <= BTM - lines; y++) {
-		memcpy(screen + y * W, screen + (y + lines) * W, W * sizeof(Cell));
+		utf_free_row(y);
+		copy_row(y, y+lines);
+		if (y < BTM - lines) utf_backup_row(y);
 	}
 
-	clear_range(y * W, (R1+1)*W-1);
+	clear_range_noutf(y * W, (BTM + 1) * W - 1);
 
-	done:
+done:
 	NOTIFY_DONE();
 }
 
@@ -828,24 +978,25 @@ screen_scroll_down(unsigned int lines)
 {
 	NOTIFY_LOCK();
 	if (lines >= RH) {
-		clear_range(R0*W, (R1+1)*W-1);
+		// clear entire region
+		clear_range_utf(TOP * W, (BTM + 1) * W - 1);
 		goto done;
 	}
 
-	// FIXME remove cleared & overwritten cells from unicode cache!
-
 	// bad cmd
 	if (lines == 0) {
 		goto done;
 	}
 
 	int y;
-	for (y = R1; y >= R0+lines; y--) {
+	for (y = BTM; y >= TOP+lines; y--) {
-		memcpy(screen + y * W, screen + (y - lines) * W, W * sizeof(Cell));
+		utf_free_row(y);
+		copy_row(y, y-lines);
+		if (y > TOP + lines) utf_backup_row(y);
 	}
 
-	clear_range(R0*W, R0*W+ lines * W-1);
+	clear_range_noutf(TOP * W, TOP * W + lines * W - 1);
-	done:
+done:
 	NOTIFY_DONE();
 }
 
@@ -921,7 +1072,7 @@ screen_cursor_get(int *y, int *x)
 	*y = cursor.y;
 
 	if (cursor.origin_mode) {
-		*y -= R0;
+		*y -= TOP;
 	}
 }
 
@@ -950,9 +1101,9 @@ screen_cursor_set_y(int y)
 {
 	NOTIFY_LOCK();
 	if (cursor.origin_mode) {
-		y += R0;
+		y += TOP;
-		if (y > R1) y = R1;
+		if (y > BTM) y = BTM;
-		if (y < R0) y = R0;
+		if (y < TOP) y = TOP;
 	} else {
 		if (y > H-1) y = H-1;
 		if (y < 0) y = 0;
@@ -973,7 +1124,7 @@ screen_cursor_move(int dy, int dx, bool scroll)
 	clear_invalid_hanging();
 
 	if (cursor.hanging && dx < 0) {
-		dx += 1; // consume one step on the removal of "xenl"
+		//dx += 1; // consume one step on the removal of "xenl"
 		cursor.hanging = false;
 	}
 
@@ -1001,7 +1152,7 @@ screen_cursor_move(int dy, int dx, bool scroll)
 					}
 					else {
 						// end of screen, end of line (wrap around)
-						cursor.y = R1;
+						cursor.y = BTM;
 						cursor.x = W - 1;
 					}
 				}
@@ -1011,10 +1162,10 @@ screen_cursor_move(int dy, int dx, bool scroll)
 		}
 	}
 
-	if (cursor.y < R0) {
+	if (cursor.y < TOP) {
 		if (was_inside) {
-			move = -(cursor.y - R0);
+			move = -(cursor.y - TOP);
-			cursor.y = R0;
+			cursor.y = TOP;
 			if (scroll) screen_scroll_down((unsigned int) move);
 		}
 		else {
@@ -1026,10 +1177,10 @@ screen_cursor_move(int dy, int dx, bool scroll)
 		}
 	}
 
-	if (cursor.y > R1) {
+	if (cursor.y > BTM) {
 		if (was_inside) {
-			move = cursor.y - R1;
+			move = cursor.y - BTM;
-			cursor.y = R1;
+			cursor.y = BTM;
 			if (scroll) screen_scroll_up((unsigned int) move);
 		}
 		else {
@@ -1360,10 +1511,10 @@ putchar_graphic(const char *ch)
 			cursor.x = 0;
 			cursor.y++;
 			// Y wrap
-			if (cursor.y > R1) {
+			if (cursor.y > BTM) {
 				// Scroll up, so we have space for writing
 				screen_scroll_up(1);
-				cursor.y = R1;
+				cursor.y = BTM;
 			}
 
 			cursor.hanging = false;

user/utf8.c

@@ -20,7 +20,8 @@ static UnicodeCacheSlot cache[UNICODE_CACHE_SIZE];
  * @return
  */
 void ICACHE_FLASH_ATTR
-unicode_cache_clear(void) {
+unicode_cache_clear(void)
+{
 	utfc_dbg("utf8 cache clear!");
 	for (int slot = 0; slot < UNICODE_CACHE_SIZE; slot++) {
 		cache[slot].count=0;
@@ -35,7 +36,8 @@ unicode_cache_clear(void) {
  * @return the obtained look-up reference
  */
 UnicodeCacheRef ICACHE_FLASH_ATTR
-unicode_cache_add(const u8 *bytes) {
+unicode_cache_add(const u8 *bytes)
+{
 	if (bytes[0] < 32) {
 		utfc_warn("utf8 cache bad char '%c'", bytes[0]);
 		return '?';
@@ -69,6 +71,27 @@ unicode_cache_add(const u8 *bytes) {
 	return ID_TO_REF(slot);
 }
 
+/**
+ * Increment a reference
+ *
+ * @param ref - reference
+ * @return success
+ */
+bool ICACHE_FLASH_ATTR
+unicode_cache_inc(UnicodeCacheRef ref)
+{
+	if (!IS_UNICODE_CACHE_REF(ref)) return true; // ASCII
+
+	int slot = REF_TO_ID(ref);
+	if (cache[slot].count == 0) {
+		utfc_warn("utf8 cache inc-after-free ref @ %d", ref);
+		return false;
+	}
+	cache[slot].count++;
+	utfc_dbg("utf8 cache inc '%.4s' @ %d, %d uses", cache[slot].bytes, slot, cache[slot].count);
+	return true;
+}
+
 /**
  * Look up a code point in the cache by reference. Do not change the use counter.
  *
@@ -77,7 +100,8 @@ unicode_cache_add(const u8 *bytes) {
  * @return true if the look-up succeeded
  */
 bool ICACHE_FLASH_ATTR
-unicode_cache_retrieve(UnicodeCacheRef ref, u8 *target) {
+unicode_cache_retrieve(UnicodeCacheRef ref, u8 *target)
+{
 	if (!IS_UNICODE_CACHE_REF(ref)) {
 		// ASCII, bypass
 		target[0] = ref;
@@ -107,7 +131,8 @@ unicode_cache_retrieve(UnicodeCacheRef ref, u8 *target) {
  * @return true if the code point was found in the cache
  */
 bool ICACHE_FLASH_ATTR
-unicode_cache_remove(UnicodeCacheRef ref) {
+unicode_cache_remove(UnicodeCacheRef ref)
+{
 	if (!IS_UNICODE_CACHE_REF(ref)) return true; // ASCII, bypass
 
 	u8 slot = REF_TO_ID(ref);

user/utf8.h

@@ -28,6 +28,14 @@ void unicode_cache_clear(void);
  */
 UnicodeCacheRef unicode_cache_add(const u8 *bytes);
 
+/**
+ * Increment a reference
+ *
+ * @param ref - reference
+ * @return success
+ */
+bool unicode_cache_inc(UnicodeCacheRef ref);
+
 /**
  * Look up a code point in the cache by reference. Do not change the use counter.
  *