Squashed 'lispBM/lispBM/' changes from bad046b0..c53ee21f

c53ee21f fix lbmref
cecd5f29 update buclear in lbmref
4c7dad51 Storing symtable list structure in flash when @const-symbol-strings is active
ad974fbf setting flash string counter to 0 at init of symrepr
dbc14c56 added an @const-symbol-string evaluator flag for storage of symbols in flash memory

git-subtree-dir: lispBM/lispBM
git-subtree-split: c53ee21fbaae7f419e096914d2bc5d8528037d88

doc/lbmref.md

@@ -1436,21 +1436,25 @@ Example that updates position 1 in a buffer:
 
 ### bufclear
 
-Clears an array by writing zeroes (or a value of choice) to all locations.
+To clear a byte array the function bufclear can be used:
-The form of a `bufclear` expression is `(bufclear buf-expr opt-val-expr)`.
-
-Example that clears a buffer:
 
 ```clj
-(bufclear buf)
+(bufclear arr optByte optStart optLen)
 ```
 
-Example that clears a buffer to all ones:
+Where arr is the byte array to clear, optByte is the optional argument
+of what to clear with (default 0), optStart is the optional argument
+of which position to start clearing (default 0) and optLen is the
+optional argument of how many bytes to clear after start (default the
+entire array). Example:
 
 ```clj
-(bufclear buf 1)
+(bufclear arr) ; Clear all of arr
+(bufclear arr 0xFF) ; Fill arr with 0xFF
+(bufclear arr 0 5) ; Clear from index 5 to the end
+(bufclear arr 0 5 10) ; Clear 10 bytes starting from index 5
+(bufclear arr 0xAA 5 10) ; Set 10 bytes to 0xAA starting from index 5
 ```
-
 ---
 
 ### Byte-array literal syntax

include/eval_cps.h

@@ -35,9 +35,10 @@ extern "C" {
 
 #define EVAL_CPS_DEFAULT_MAILBOX_SIZE 10
 
-#define EVAL_CPS_CONTEXT_FLAG_NOTHING       (uint32_t)0x0
+#define EVAL_CPS_CONTEXT_FLAG_NOTHING               (uint32_t)0x0
-#define EVAL_CPS_CONTEXT_FLAG_TRAP          (uint32_t)0x1
+#define EVAL_CPS_CONTEXT_FLAG_TRAP                  (uint32_t)0x1
-#define EVAL_CPS_CONTEXT_FLAG_CONST         (uint32_t)0x2
+#define EVAL_CPS_CONTEXT_FLAG_CONST                 (uint32_t)0x2
+#define EVAL_CPS_CONTEXT_FLAG_CONST_SYMBOL_STRINGS  (uint32_t)0x4
   
 /** The eval_context_t struct represents a lispbm process.
  *

include/symrepr.h

@@ -58,6 +58,13 @@ void lbm_symrepr_name_iterator(symrepr_name_iterator_fun f);
  * \return 1 for success and 0 for failure.
  */
 int lbm_add_symbol(char *name, lbm_uint *id);
+/** Add a symbol to the symbol table. The symbol name string is copied to flash.
+ *
+ * \param name String representation of the symbol.
+ * \param id Resulting id is returned through this argument.
+ * \return 1 for success and 0 for failure.
+ */
+int lbm_add_symbol_flash(char *name, lbm_uint* id);
 /** Name of symbol to symbol. If the symbol exists the ID of the symbol is returned.
     If the name does not match any existing symbol, one is created and that ID is returned.
     \param name String name of symbol.
@@ -120,13 +127,22 @@ int lbm_get_num_variables(void);
 
 /**
  *
- * \return The total amount of space occupied by the symbol table in bytes.
+ * \return The total amount of lbm_memory space occupied by the symbol table in bytes.
  */
 lbm_uint lbm_get_symbol_table_size(void);
+/**
+ *
+ * \return The total amount of flash space occupied by the symbol table in bytes.
+ */
+lbm_uint lbm_get_symbol_table_size_flash(void);
 /**
  * \return The size in bytes of all symbol strings stored in the symbol table.
  */
 lbm_uint lbm_get_symbol_table_size_names(void);
+/**
+ * \return The size in bytes of all symbol strings stored in flash from the symbol table.
+ */
+lbm_uint lbm_get_symbol_table_size_names_flash(void);
 
 #ifdef __cplusplus
 }

include/tokpar.h

@@ -37,6 +37,7 @@
 #define TOKCLOSECURL    13u     // "}"
 #define TOKCONSTSTART   14u     // "@const-start"
 #define TOKCONSTEND     15u     // "@const-end"
+#define TOKCONSTSYMSTR  16u     // "@const-symbol-strings"
 
 #define TOKTYPEBYTE     100u
 #define TOKTYPEI        101u

repl/repl.c

@@ -215,7 +215,6 @@ void sleep_callback(uint32_t us) {
 
 
 bool dyn_load(const char *str, const char **code) {
-
   bool res = false;
   if (strlen(str) == 5 && strncmp(str, "defun", 5) == 0) {
     *code = "(define defun (macro (name args body) `(define ,name (lambda ,args ,body))))";
@@ -640,8 +639,10 @@ int main(int argc, char **argv) {
       printf("Memory size: %"PRI_UINT" Words\n", lbm_memory_num_words());
       printf("Memory free: %"PRI_UINT" Words\n", lbm_memory_num_free());
       printf("Allocated arrays: %"PRI_UINT"\n", heap_state.num_alloc_arrays);
-      printf("Symbol table size: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size());
+      printf("Symbol table size RAM: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size());
-      printf("Symbol names size: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size_names());
+      printf("Symbol names size RAM: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size_names());
+      printf("Symbol table size FLASH: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size_flash());
+      printf("Symbol names size FLASH: %"PRI_UINT" Bytes\n", lbm_get_symbol_table_size_names_flash());
       free(str);
     }  else if (strncmp(str, ":env", 4) == 0) {
       lbm_value curr = *lbm_get_env_ptr();

src/eval_cps.c

@@ -1,4 +1,4 @@
- /*
+/*
     Copyright 2018, 2020, 2021, 2022, 2023 Joel Svensson    svenssonjoel@yahoo.se
 
     This program is free software: you can redistribute it and/or modify
@@ -170,7 +170,6 @@ typedef struct {
   eval_context_t *last;
 } eval_context_queue_t;
 
-
 static int gc(void);
 void error_ctx(lbm_value);
 static void enqueue_ctx(eval_context_queue_t *q, eval_context_t *ctx);
@@ -658,6 +657,28 @@ static void block_current_ctx(lbm_uint sleep_us, uint32_t wait_mask, bool do_con
   ctx_running = NULL;
 }
 
+lbm_flash_status lbm_write_const_array_padded(uint8_t *data, lbm_uint n, lbm_uint *res) {
+  lbm_uint full_words = n / sizeof(lbm_uint);
+  lbm_uint n_mod = n % sizeof(lbm_uint);
+
+  if (n_mod == 0) { // perfect fit.
+    return lbm_write_const_raw((lbm_uint*)data, full_words, res);
+  } else {
+    lbm_uint last_word = 0;
+    memcpy(&last_word, &data[full_words * sizeof(lbm_uint)], n_mod);
+    if (full_words >= 1) {
+      lbm_flash_status s = lbm_write_const_raw((lbm_uint*)data, full_words, res);
+      if ( s == LBM_FLASH_WRITE_OK) {
+        lbm_uint dummy;
+        s = lbm_write_const_raw(&last_word, 1, &dummy);
+      }
+      return s;
+    } else {
+      return lbm_write_const_raw(&last_word, 1, res);
+    }
+  }
+}
+
 /****************************************************/
 /* Error message creation                           */
 
@@ -2957,6 +2978,13 @@ static void cont_read_next_token(eval_context_t *ctx) {
       stack_push(&ctx->K, READ_NEXT_TOKEN);
       ctx->app_cont = true;
       return;
+    case TOKCONSTSYMSTR:
+      ctx->flags |= EVAL_CPS_CONTEXT_FLAG_CONST_SYMBOL_STRINGS;
+      sptr[0] = stream;
+      sptr[1] = lbm_enc_u(0);
+      stack_push(&ctx->K, READ_NEXT_TOKEN);
+      ctx->app_cont = true;
+      return;
     default:
       read_error_ctx(lbm_channel_row(chan), lbm_channel_column(chan));
     }
@@ -3063,15 +3091,18 @@ static void cont_read_next_token(eval_context_t *ctx) {
 
     if (lbm_get_symbol_by_name(tokpar_sym_str, &symbol_id)) {
       res = lbm_enc_sym(symbol_id);
-    }
+    } else {
-    else {
       int r = 0;
       if (strncmp(tokpar_sym_str,"ext-",4) == 0) {
         r = lbm_add_extension_symbol(tokpar_sym_str, &symbol_id);
       } else if (tokpar_sym_str[0] == '#') {
         r = lbm_add_variable_symbol(tokpar_sym_str, &symbol_id);
       } else {
-        r = lbm_add_symbol(tokpar_sym_str, &symbol_id);
+        if (ctx->flags & EVAL_CPS_CONTEXT_FLAG_CONST_SYMBOL_STRINGS) {
+          r = lbm_add_symbol_flash(tokpar_sym_str, &symbol_id);
+        } else {
+          r = lbm_add_symbol(tokpar_sym_str, &symbol_id);
+        }
       }
       if (r) {
         res = lbm_enc_sym(symbol_id);
@@ -3656,26 +3687,9 @@ static void cont_move_val_to_flash_dispatch(eval_context_t *ctx) {
       } break;
       case SYM_ARRAY_TYPE: {
         lbm_array_header_t *arr = (lbm_array_header_t*)ref->car;
-
-        lbm_uint full_words = arr->size / sizeof(lbm_uint); // array size always in bytes
-
         // arbitrary address: flash_arr.
         lbm_uint flash_arr;
-        if ( arr->size % sizeof(lbm_uint) == 0) {
+        handle_flash_status(lbm_write_const_array_padded((uint8_t*)arr->data, arr->size, &flash_arr));
-          handle_flash_status(lbm_write_const_raw(arr->data, full_words, &flash_arr));
-        } else {
-          lbm_uint last_word = 0;
-          memcpy(&last_word, &arr->data[full_words],arr->size % sizeof(lbm_uint));
-
-          if (full_words >= 1) {
-            handle_flash_status(lbm_write_const_raw(arr->data, full_words, &flash_arr));
-            lbm_uint dummy;
-            handle_flash_status(lbm_write_const_raw(&last_word, 1, &dummy));
-          } else {
-            handle_flash_status(lbm_write_const_raw(&last_word, 1, &flash_arr));
-          }
-        }
-
         lift_array_flash(flash_cell,
                          (char *)flash_arr,
                          arr->size);

src/symrepr.c

@@ -20,8 +20,9 @@
 #include <string.h>
 #include <stdlib.h>
 #include <inttypes.h>
-#include <lbm_memory.h>
 
+#include <lbm_memory.h>
+#include <heap.h>
 #include "symrepr.h"
 
 #define NUM_SPECIAL_SYMBOLS (sizeof(special_symbols) / sizeof(special_sym))
@@ -222,9 +223,9 @@ static lbm_uint next_extension_symbol_id = EXTENSION_SYMBOLS_START;
 static lbm_uint next_variable_symbol_id = VARIABLE_SYMBOLS_START;
 
 static lbm_uint symbol_table_size_list = 0;
+static lbm_uint symbol_table_size_list_flash = 0;
 static lbm_uint symbol_table_size_strings = 0;
-
+static lbm_uint symbol_table_size_strings_flash = 0;
-
 
 int lbm_symrepr_init(void) {
   symlist = NULL;
@@ -232,7 +233,9 @@ int lbm_symrepr_init(void) {
   next_extension_symbol_id = EXTENSION_SYMBOLS_START;
   next_variable_symbol_id = VARIABLE_SYMBOLS_START;
   symbol_table_size_list = 0;
+  symbol_table_size_list_flash = 0;
   symbol_table_size_strings = 0;
+  symbol_table_size_strings_flash = 0;
   return 1;
 }
 
@@ -292,7 +295,9 @@ int lbm_get_symbol_by_name(char *name, lbm_uint* id) {
   return 0;
 }
 
-static bool store_symbol_name(char *name, lbm_uint *res) {
+extern lbm_flash_status lbm_write_const_array_padded(uint8_t *data, lbm_uint n, lbm_uint *res);
+
+static bool store_symbol_name_base(char *name, lbm_uint *res, bool flash) {
   size_t n = strlen(name) + 1;
   if (n == 1) return 0; // failure if empty symbol
 
@@ -303,17 +308,31 @@ static bool store_symbol_name(char *name, lbm_uint *res) {
   } else {
     alloc_size = (n/(sizeof(lbm_uint))) + 1;
   }
+  if (flash) {
+    lbm_uint symbol_addr = 0;
+    lbm_flash_status s = lbm_write_const_array_padded((uint8_t*)name, n, &symbol_addr);
+    if (s != LBM_FLASH_WRITE_OK || symbol_addr == 0) {
+      return false;
+    }
+    symbol_table_size_strings_flash += alloc_size;
+    *res = symbol_addr;
+    return true;
+  } else {
+    symbol_name_storage = (char *)lbm_memory_allocate(alloc_size);
+    if (symbol_name_storage == NULL) return false;
+    symbol_table_size_strings += alloc_size;
+    strcpy(symbol_name_storage, name);
+    *res = (lbm_uint)symbol_name_storage;
+    return true;
+  }
+}
 
-  symbol_name_storage = (char *)lbm_memory_allocate(alloc_size);
+static bool store_symbol_name(char *name, lbm_uint *res) {
+  return store_symbol_name_base(name, res, false);
+}
 
-  if (symbol_name_storage == NULL) return false;
+static bool store_symbol_name_flash(char *name, lbm_uint *res) {
-
+  return store_symbol_name_base(name, res, true);
-  symbol_table_size_strings += alloc_size;
-
-  strcpy(symbol_name_storage, name);
-
-  *res = (lbm_uint)symbol_name_storage;
-  return true;
 }
 
 static bool add_symbol_to_symtab(lbm_uint name, lbm_uint id) {
@@ -323,32 +342,50 @@ static bool add_symbol_to_symtab(lbm_uint name, lbm_uint id) {
 
   symbol_table_size_list += 3;
   m[NAME] = name;
-
+  m[NEXT] = (lbm_uint) symlist;
-  if (symlist == NULL) {
+  symlist = m;
-    m[NEXT] = (lbm_uint) NULL;
-    symlist = m;
-  } else {
-    m[NEXT] = (lbm_uint) symlist;
-    symlist = m;
-  }
   m[ID] =id;
   return true;
 }
 
-int lbm_add_symbol(char *name, lbm_uint* id) {
+static bool add_symbol_to_symtab_flash(lbm_uint name, lbm_uint id) {
-
+  lbm_uint entry[3];
-  lbm_uint symbol_name_storage;
+  entry[NAME] = name;
-  if (!store_symbol_name(name, &symbol_name_storage)) return 0;
+  entry[NEXT] = (lbm_uint) symlist;
-
+  entry[ID]   = id;
-  if (!add_symbol_to_symtab(symbol_name_storage, next_symbol_id)) {
+  lbm_uint entry_addr = 0;
-    lbm_memory_free((lbm_uint*)symbol_name_storage);
+  if (lbm_write_const_raw(entry,3, &entry_addr) == LBM_FLASH_WRITE_OK) {
-    return 0;
+    symlist = (lbm_uint*)entry_addr;
+    symbol_table_size_list_flash += 3;
+    return true;
   }
+  return false;
+}
 
+static int lbm_add_symbol_base(char *name, lbm_uint *id, bool flash) {
+  lbm_uint symbol_name_storage;
+  if (flash) {
+    if (!store_symbol_name_flash(name, &symbol_name_storage)) return 0;
+    if (!add_symbol_to_symtab_flash(symbol_name_storage, next_symbol_id)) return 0;
+  } else {
+    if (!store_symbol_name(name, &symbol_name_storage)) return 0;
+    if (!add_symbol_to_symtab(symbol_name_storage, next_symbol_id)) {
+      lbm_memory_free((lbm_uint*)symbol_name_storage);
+      return 0;
+    }
+  }
   *id = next_symbol_id ++;
   return 1;
 }
 
+int lbm_add_symbol(char *name, lbm_uint* id) {
+  return lbm_add_symbol_base(name, id, false);
+}
+
+int lbm_add_symbol_flash(char *name, lbm_uint* id) {
+  return lbm_add_symbol_base(name, id, true);
+}
+
 int lbm_add_symbol_const(char *name, lbm_uint* id) {
 
   if (!add_symbol_to_symtab((lbm_uint)name, next_symbol_id)) {
@@ -431,10 +468,19 @@ lbm_uint lbm_get_symbol_table_size(void) {
           symbol_table_size_strings) * sizeof(lbm_uint);
 }
 
+lbm_uint lbm_get_symbol_table_size_flash(void) {
+  return (symbol_table_size_list_flash +
+          symbol_table_size_strings_flash) * sizeof(lbm_uint);
+}
+
 lbm_uint lbm_get_symbol_table_size_names(void) {
   return symbol_table_size_strings * sizeof(lbm_uint);
 }
 
+lbm_uint lbm_get_symbol_table_size_names_flash(void) {
+  return symbol_table_size_strings_flash * sizeof(lbm_uint);
+}
+
 int lbm_get_num_variables(void) {
   return (int)next_variable_symbol_id - VARIABLE_SYMBOLS_START;
 }

src/tokpar.c

@@ -43,7 +43,7 @@ typedef struct {
 } matcher;
 
 
-#define NUM_FIXED_SIZE_TOKENS 15
+#define NUM_FIXED_SIZE_TOKENS 16
 const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS] = {
   {"(", TOKOPENPAR, 1},
   {")", TOKCLOSEPAR, 1},
@@ -59,7 +59,8 @@ const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS] = {
   {"{", TOKOPENCURL, 1},
   {"}", TOKCLOSECURL, 1},
   {"@const-start", TOKCONSTSTART, 12},
-  {"@const-end", TOKCONSTEND, 10}, 
+  {"@const-end", TOKCONSTEND, 10},
+  {"@const-symbol-strings", TOKCONSTSYMSTR, 21},
 };
 
 #define NUM_TYPE_QUALIFIERS 9
@@ -155,6 +156,7 @@ int tok_symbol(lbm_char_channel_t *chan) {
     r = lbm_channel_peek(chan,(unsigned int)len, &c);
   }
   if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
+  tokpar_sym_str[len] = 0;
   return len;
 }