some fixes

include/fh_input.h

@@ -15,15 +15,22 @@ typedef bool (*fh_input_refill_t)(struct fh_thread_s *fh, struct fh_input_spec_s
 /** Spec free func */
 typedef void (*fh_input_free_t)(void *spec);
 
+typedef void (*fh_input_start_t)(struct fh_thread_s *fh, struct fh_input_spec_s *spec);
+
+typedef void (*fh_input_pushpop_t)(struct fh_thread_s *fh, struct fh_input_spec_s *spec);
+
 struct fh_input_spec_s {
   struct fh_input_spec_s *previous;
   fh_input_refill_t refill_input_buffer;
+  fh_input_start_t start;
   fh_input_free_t free_self;
+  fh_input_pushpop_t push_prepare;
+  fh_input_pushpop_t pop_restore;
   uint32_t linenum;
   char *cwd; // CWD of the input, used for relative includes. malloc'd (strdup)
 
   // saved values, filled when pushing
-  char saved_buffer[INPUT_BUFFER_SIZE];
+  uint32_t saved_inputaddr;
   uint32_t saved_inputptr;
   uint32_t saved_inputlen;
   uint32_t saved_execptr;

include/fh_runtime.h

@@ -193,6 +193,8 @@ struct fh_thread_s {
   /** Pictured numeric output buffer write cursor (used for prepend) */
   uint32_t pictnumptr;
 
+  /** Start addr of the input string (SOURCE) */
+  uint32_t inputaddr;
   /** Input buffer parse position */
   uint32_t inputptr;
   /** Input buffer total content size */

src/fh_builtins_meta.c

@@ -295,10 +295,8 @@ static enum fh_error w_source(struct fh_thread_s *fh, const struct fh_word_s *w)
 {
   (void) w;
   enum fh_error rv;
-  TRY(ds_push(fh, INPUTBUF_ADDR));
+  TRY(ds_push(fh, fh->inputaddr));
   TRY(ds_push(fh, fh->inputlen));
-//  TRY(ds_push(fh, INPUTBUF_ADDR + fh->inputptr));
-//  TRY(ds_push(fh, fh->inputlen - fh->inputptr));
   return FH_OK;
 }
 
@@ -601,7 +599,6 @@ static enum fh_error w_evaluate(struct fh_thread_s *fh, const struct fh_word_s *
   enum fh_error rv;
   uint32_t addr, count;
   TRY(ds_pop_addr_len(fh, &addr, &count));
-
   fh_runtime_start(fh, fh_create_input_from_string(fh_str_at(fh, addr), count, fh->input->cwd));
   return FH_OK;
 }

src/fh_input.c

@@ -13,13 +13,17 @@ enum fh_error fh_push_input(struct fh_thread_s *fh, struct fh_input_spec_s *newi
   fh->input = NULL;
 
   if (NULL != oldinput) {
-    memcpy(&oldinput->saved_buffer[0], &fh->heap[INPUTBUF_ADDR], INPUT_BUFFER_SIZE);
+    oldinput->push_prepare(fh, oldinput);
+
+//    memcpy(&oldinput->saved_buffer[0], &fh->heap[INPUTBUF_ADDR], INPUT_BUFFER_SIZE);
     oldinput->saved_inputlen = fh->inputlen;
     oldinput->saved_inputptr = fh->inputptr;
     oldinput->saved_execptr = fh->execptr;
+    oldinput->saved_inputaddr = fh->inputaddr;
     newinput->previous = oldinput;
   }
 
+  fh->execptr = MAGICADDR_EXEC_INTERACTIVE;
   fh->inputlen = 0;
   fh->inputptr = 0;
 
@@ -46,10 +50,13 @@ enum fh_error fh_pop_input(struct fh_thread_s *fh)
   }
   fh->input = restored; // this can be NULL, that must be checked by caller.
 
-  memcpy(&fh->heap[INPUTBUF_ADDR], &restored->saved_buffer[0], INPUT_BUFFER_SIZE);
+  restored->pop_restore(fh, restored);
+
+//  memcpy(&fh->heap[INPUTBUF_ADDR], &restored->saved_buffer[0], INPUT_BUFFER_SIZE);
   fh->inputlen = restored->saved_inputlen;
   fh->inputptr = restored->saved_inputptr;
   fh->execptr = restored->saved_execptr;
+  fh->inputaddr = restored->saved_inputaddr;
 //  fh_setstate(fh, restored->saved_state, 0);
 
   if (discarded->free_self) {
@@ -65,6 +72,7 @@ enum fh_error fh_pop_input(struct fh_thread_s *fh)
 
 struct file_input_spec {
   struct fh_input_spec_s spec;
+  char saved_buffer[INPUT_BUFFER_SIZE];
   FILE *file;
 };
 
@@ -72,12 +80,31 @@ struct string_input_spec {
   struct fh_input_spec_s spec;
   const char *str;
   size_t len;
-  size_t readpos;
 };
 
+static void file_push_prepare(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+  struct file_input_spec *fspec = (void*) spec;
+  memcpy(&fspec->saved_buffer[0], &fh->heap[INPUTBUF_ADDR], INPUT_BUFFER_SIZE);
+}
+
+static void file_pop_restore(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+  struct file_input_spec *fspec = (void*) spec;
+  memcpy(&fh->heap[INPUTBUF_ADDR], &fspec->saved_buffer[0], INPUT_BUFFER_SIZE);
+}
+
+static void str_push_prepare(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+}
+
+static void str_pop_restore(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+}
+
 static inline uint8_t *inputbuf_at(struct fh_thread_s *fh, size_t pos)
 {
-  return &fh->heap[INPUTBUF_ADDR + pos];
+  return &fh->heap[fh->inputaddr + pos];
 }
 
 void fh_input_memmove_leftovers(struct fh_thread_s *fh)
@@ -86,33 +113,45 @@ void fh_input_memmove_leftovers(struct fh_thread_s *fh)
     // something is left
     uint32_t remains = fh->inputlen - fh->inputptr;
     if (remains > 0) {
-      LOG("Refill, reuse %d bytes left in buffer", remains);
       memmove(inputbuf_at(fh, 0), inputbuf_at(fh, fh->inputptr), remains);
       fh->inputptr = 0;
       fh->inputlen = remains;
     } else {
-      LOG("Refill, nothing reused (1)");
       fh->inputptr = 0;
       fh->inputlen = 0;
     }
   } else {
-    LOG("Refill, nothing reused (2)");
     fh->inputptr = 0;
     fh->inputlen = 0;
   }
 }
 
+static void file_start(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+  fh->inputaddr = INPUTBUF_ADDR;
+  //
+}
+
+static void str_start(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
+{
+  struct string_input_spec *strspec = (struct string_input_spec *) spec;
+  fh->inputaddr = (void*)strspec->str - (void*)&fh->heap[0]; // XXX this will explode if string is not in the internal heap!
+  fh->inputlen = strspec->len;
+  fh->inputptr = 0;
+}
+
 static bool file_refill(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
 {
   struct file_input_spec *fis = (struct file_input_spec *) spec;
   fh_input_memmove_leftovers(fh);
   uint32_t space_left = INPUT_BUFFER_SIZE - fh->inputlen - 1;
   char *wp = (char *) inputbuf_at(fh, fh->inputptr);
-  LOG("spec %p, fgets %d", spec, space_left);
+  //LOG("spec %p, fgets %d", spec, space_left);
   if (fgets(wp, (int) space_left, fis->file)) {
     spec->linenum++;
     fh->inputlen = strnlen(wp, INPUT_BUFFER_SIZE);
-    LOG("read %d bytes from file", fh->inputlen);
+    wp[fh->inputlen] = 0;
+    //LOG("read %d bytes from file", fh->inputlen);
     return true;
   } else {
     return fh->inputptr > fh->inputlen; // return false only if there is nothing left
@@ -121,25 +160,26 @@ static bool file_refill(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
 
 static bool str_refill(struct fh_thread_s *fh, struct fh_input_spec_s *spec)
 {
-  struct string_input_spec *fis = (struct string_input_spec *) spec;
-  fh_input_memmove_leftovers(fh);
-  uint32_t space_left = INPUT_BUFFER_SIZE - fh->inputlen - 1;
-  char *wp = (char *) inputbuf_at(fh, fh->inputptr);
-
-  uint32_t chars_remaining_in_string = fis->len - fis->readpos;
-  if (chars_remaining_in_string > 0) {
-    if (chars_remaining_in_string < space_left) {
-      space_left = chars_remaining_in_string;
-    }
-
-    memcpy(wp, &fis->str[fis->readpos], space_left);
-    *(wp + space_left) = 0;
-    fis->readpos += space_left;
-    fh->inputlen += space_left;
-    return true;
-  } else {
-    return false;
-  }
+  return fh->inputptr < fh->inputlen;
+//  struct string_input_spec *fis = (struct string_input_spec *) spec;
+//  fh_input_memmove_leftovers(fh);
+//  uint32_t space_left = INPUT_BUFFER_SIZE - fh->inputlen - 1;
+//  char *wp = (char *) inputbuf_at(fh, fh->inputptr);
+//
+//  uint32_t chars_remaining_in_string = fis->len - fis->readpos;
+//  if (chars_remaining_in_string > 0) {
+//    if (chars_remaining_in_string < space_left) {
+//      space_left = chars_remaining_in_string;
+//    }
+//
+//    memcpy(wp, &fis->str[fis->readpos], space_left);
+//    *(wp + space_left) = 0;
+//    fis->readpos += space_left;
+//    fh->inputlen += space_left;
+//    return true;
+//  } else {
+//    return false;
+//  }
 }
 
 static void free_filespec(void *p)
@@ -168,19 +208,23 @@ static void free_strspec(void *p)
 
 struct fh_input_spec_s *fh_create_input_from_filestruct(FILE *f, const char *cwd)
 {
-  struct file_input_spec *fis = calloc(sizeof(struct file_input_spec), 1);
-  if (!fis) {
+  struct file_input_spec *spec = calloc(sizeof(struct file_input_spec), 1);
+  if (!spec) {
     LOGE("Err alloc input spec struct");
     return NULL;
   }
 
-  fis->spec.free_self = free_filespec;
-  fis->spec.refill_input_buffer = file_refill;
-  fis->file = f;
+  LOG("Input for FILE, cwd: %s", cwd);
+  spec->spec.free_self = free_filespec;
+  spec->spec.start = file_start;
+  spec->spec.push_prepare = file_push_prepare;
+  spec->spec.pop_restore = file_pop_restore;
+  spec->spec.refill_input_buffer = file_refill;
+  spec->file = f;
   if (cwd) {
-    fis->spec.cwd = strdup(cwd);
+    spec->spec.cwd = strdup(cwd);
   }
-  return (struct fh_input_spec_s*) fis;
+  return (struct fh_input_spec_s*) spec;
 }
 
 struct fh_input_spec_s *fh_create_input_from_filename(char *path)
@@ -199,6 +243,9 @@ struct fh_input_spec_s *fh_create_input_from_filename(char *path)
   }
 
   spec->spec.free_self = free_filespec;
+  spec->spec.start = file_start;
+  spec->spec.push_prepare = file_push_prepare;
+  spec->spec.pop_restore = file_pop_restore;
   spec->spec.refill_input_buffer = file_refill;
   spec->file = f;
 
@@ -216,18 +263,22 @@ struct fh_input_spec_s *fh_create_input_from_filename(char *path)
 
 struct fh_input_spec_s *fh_create_input_from_string(char *str, size_t len, const char *cwd)
 {
-  struct string_input_spec *sis = calloc(sizeof(struct string_input_spec), 1);
-  if (!sis) {
+  struct string_input_spec *spec = calloc(sizeof(struct string_input_spec), 1);
+  if (!spec) {
     return NULL;
   }
 
-  sis->spec.free_self = free_strspec;
-  sis->spec.refill_input_buffer = str_refill;
-  sis->str = str;
-  sis->readpos = 0;
-  sis->len = len;
-  sis->spec.cwd = cwd ? strdup(cwd) : NULL;
-  return (struct fh_input_spec_s*) sis;
+  LOG("Evaluating: \"%.*s\"", (int)len, str);
+
+  spec->spec.free_self = free_strspec;
+  spec->spec.refill_input_buffer = str_refill;
+  spec->spec.push_prepare = str_push_prepare;
+  spec->spec.pop_restore = str_pop_restore;
+  spec->spec.start = str_start;
+  spec->str = str;
+  spec->len = len;
+  spec->spec.cwd = cwd ? strdup(cwd) : NULL;
+  return (struct fh_input_spec_s*) spec;
 }
 
 void fh_input_teardown(struct fh_thread_s *fh)

src/fh_mem.c

@@ -3,15 +3,6 @@
 // Important distinction: HEAP_END is the end of the normally addressable region. HEAP_SIZE is the full memory area.
 //  Buffers are placed at the end of the heap!
 
-void fh_fill_input_buffer(struct fh_thread_s *fh, const char *data, size_t num)
-{
-  assert(num <= INPUT_BUFFER_SIZE);
-  memcpy(&fh->heap[INPUTBUF_ADDR], data, num);
-  fh->heap[INPUTBUF_ADDR + num] = 0; // terminator
-  fh->inputptr = 0;
-  fh->inputlen = num;
-}
-
 void fh_align(struct fh_thread_s *fh)
 {
   fh->here = WORDALIGNED(fh->here);

src/fh_parse.c

@@ -63,7 +63,7 @@ enum fh_error fh_handle_ascii_word(
 
 void fh_input_consume_matching(struct fh_thread_s *fh, chartest_t test, void *param)
 {
-  char *rp = (char *) &fh->heap[INPUTBUF_ADDR + fh->inputptr];
+  char *rp = (char *) &fh->heap[fh->inputaddr + fh->inputptr];
   while (test(*rp, param)) {
     rp++;
     fh->inputptr++;
@@ -72,7 +72,7 @@ void fh_input_consume_matching(struct fh_thread_s *fh, chartest_t test, void *pa
 
 void fh_input_consume_spaces(struct fh_thread_s *fh)
 {
-  char *rp = (char *) &fh->heap[INPUTBUF_ADDR + fh->inputptr];
+  char *rp = (char *) &fh->heap[fh->inputaddr + fh->inputptr];
   while (isspace(*rp)) {
     rp++;
     fh->inputptr++;
@@ -81,9 +81,9 @@ void fh_input_consume_spaces(struct fh_thread_s *fh)
 
 enum fh_error fh_input_read_delimited(struct fh_thread_s *fh, char **out, size_t *len, chartest_t test, void *param)
 {
-  char *rp = (char *) &fh->heap[INPUTBUF_ADDR + fh->inputptr];
+  char *rp = (char *) &fh->heap[fh->inputaddr + fh->inputptr];
   char *start = rp;
-  while (1) {
+  while (fh->inputptr < fh->inputlen) {
     char c = *rp;
     if (test(c, param)) {
       if (rp == start) {
@@ -98,6 +98,9 @@ enum fh_error fh_input_read_delimited(struct fh_thread_s *fh, char **out, size_t
     rp++;
     fh->inputptr++;
   }
+  *out = start;
+  *len = rp - start;
+  return FH_OK;
 }
 
 static bool chartest_space_or_end(char c, void *param)
@@ -113,7 +116,7 @@ enum fh_error fh_input_read_word(struct fh_thread_s *fh, char **out, size_t *len
 
 enum fh_error fh_input_read_quotedstring(struct fh_thread_s *fh, bool escaped, char *outbuf, size_t capacity, size_t *out_len)
 {
-  char *rp = (char *) &fh->heap[INPUTBUF_ADDR + fh->inputptr];
+  char *rp = (char *) &fh->heap[fh->inputaddr + fh->inputptr];
   bool next_escaped = false;
   size_t remains = capacity;
   size_t len = 0;
@@ -233,12 +236,14 @@ enum fh_error fh_runtime_start(struct fh_thread_s *fh, struct fh_input_spec_s *i
     FHPRINT("%s", FH_PROMPT_STR);
   }
 
+  input->start(fh, input);
+
   while (1) {
-    LOG("Refill input buffer");
+    //LOG("Refill input buffer");
     if (fh->input->refill_input_buffer(fh, fh->input)) {
       // discard spaces at the end
-      while (isspace(fh->heap[INPUTBUF_ADDR + fh->inputlen - 1]) && fh->inputlen > 0) {
-        fh->heap[INPUTBUF_ADDR + fh->inputlen - 1] = 0;
+      while (isspace(fh->heap[fh->inputaddr + fh->inputlen - 1]) && fh->inputlen > 0) {
+        fh->heap[fh->inputaddr + fh->inputlen - 1] = 0;
         fh->inputlen--;
       }
 
@@ -291,7 +296,7 @@ enum fh_error fh_process_line(struct fh_thread_s *fh)
 {
   enum fh_error rv;
 
-#define ReadPtr ((char*)(&fh->heap[INPUTBUF_ADDR + fh->inputptr]))
+#define ReadPtr ((char*)(&fh->heap[fh->inputaddr + fh->inputptr]))
 #define ReadPos (fh->inputptr)
 #define ReadLen (fh->inputlen)
 

testfiles/combinedtest.f

@@ -1049,6 +1049,7 @@ TESTING OUTPUT: . ." CR EMIT SPACE SPACES TYPE U.
 
 T{ OUTPUT-TEST -> }T
 
+0 [if]
 \ ------------------------------------------------------------------------
 TESTING INPUT: ACCEPT
 
@@ -1062,6 +1063,7 @@ CREATE ABUF 50 CHARS ALLOT
 ;
 
 T{ ACCEPT-TEST -> }T
+[then]
 
 \ ------------------------------------------------------------------------
 TESTING DICTIONARY SEARCH RULES
@@ -1586,6 +1588,7 @@ T{ 25 RN2 EXECUTE -> 33 22 11 0 }T
 \ -----------------------------------------------------------------------------
 TESTING C"
 
+1 debug
 T{ : CQ1 C" 123" ; -> }T
 T{ CQ1 COUNT EVALUATE -> 123 }T
 T{ : CQ2 C" " ; -> }T

testfiles/rescan.f

@@ -0,0 +1,7 @@
+VARIABLE SCANS
+: RESCAN?  -1 SCANS +! SCANS @ IF 0 >IN ! THEN ;
+
+1 debug
+: GS2 1 SCANS ! S" 123 RESCAN?" EVALUATE ;
+GS2
+??