rusefi
/
bldc
mirror of https://github.com/rusefi/bldc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Squashed 'lispBM/lispBM/' changes from 60d7c83f..c0fb4af8 

c0fb4af8 Streaming reading or source into the heap
540f1e9a added n-times test for low-freq concurrency bug testing
5dcf25bf update chibios examples to use new reader
7775f92d starting to look good, needs more polish...
86ecd85b On the path
b0ce4e67 tweaks to next_token
c7ed1e3d Update and discovery of nondeterministicly appearing bug
a0a1b000 Ok start
090f54e9 experiment with channel abstraction
d4c1ba41 added some tests of array syntax

git-subtree-dir: lispBM/lispBM
git-subtree-split: c0fb4af806ad8d42365a1b8695d2603f640df662
This commit is contained in:
Benjamin Vedder 2022-09-16 09:25:46 +02:00
parent 048b9469d3
commit 9bd826e646
39 changed files with 1250 additions and 1754 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
benchmarks/bench_chibi/Makefile
View File

@ -122,8 +122,7 @@ LDSCRIPT= $(STARTUPLD)/STM32F407xG.ld
# C sources that can be compiled in ARM or THUMB mode depending on the global
# setting.
LBMSRC = ../../src/compression.c \
../../src/env.c \
LBMSRC = ../../src/env.c \
../../src/eval_cps.c \
../../src/extensions.c \
../../src/fundamental.c \
@ -132,13 +131,13 @@ LBMSRC = ../../src/compression.c \
../../src/print.c \
../../src/qq_expand.c \
../../src/stack.c \
../../src/streams.c \
../../src/symrepr.c \
../../src/tokpar.c \
../../src/lispbm.c \
../../src/lbm_c_interop.c \
../../src/lbm_variables.c \
../../src/lbm_custom_type.c \
../../src/lbm_channel.c \
../../platform/chibios/src/platform_mutex.c
CSRC = $(ALLCSRC) \

16
benchmarks/bench_chibi/main.c
View File

@ -53,8 +53,8 @@ static lbm_cons_t heap[HEAP_SIZE] __attribute__ ((aligned (8)));
static uint32_t memory_array[LBM_MEMORY_SIZE_8K];
static uint32_t bitmap_array[LBM_MEMORY_BITMAP_SIZE_8K];
static lbm_tokenizer_string_state_t string_tok_state;
static lbm_tokenizer_char_stream_t string_tok;
static lbm_string_channel_state_t string_tok_state;
static lbm_char_channel_t string_tok;
BaseSequentialStream *chp = NULL;
@ -362,9 +362,9 @@ int main(void) {
while(lbm_get_eval_state() != EVAL_CPS_STATE_PAUSED) {
sleep_callback(10);
}
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
file_buffer);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
file_buffer);
systime_t t_load = chVTGetSystemTimeX();
@ -413,9 +413,9 @@ int main(void) {
sleep_callback(10);
}
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
str);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
str);
lbm_cid cid = lbm_load_and_eval_expression(&string_tok);

5
chibios-examples/repl-ChibiOS/Makefile
View File

@ -122,8 +122,7 @@ LDSCRIPT= $(STARTUPLD)/STM32F407xG.ld
# C sources that can be compiled in ARM or THUMB mode depending on the global
# setting.
LBMSRC = ../../src/compression.c \
../../src/env.c \
LBMSRC = ../../src/env.c \
../../src/eval_cps.c \
../../src/extensions.c \
../../src/fundamental.c \
@ -132,13 +131,13 @@ LBMSRC = ../../src/compression.c \
../../src/print.c \
../../src/qq_expand.c \
../../src/stack.c \
../../src/streams.c \
../../src/symrepr.c \
../../src/tokpar.c \
../../src/lispbm.c \
../../src/lbm_c_interop.c \
../../src/lbm_variables.c \
../../src/lbm_custom_type.c \
../../src/lbm_channel.c \
../../src/extensions/array_extensions.c \
../../src/extensions/string_extensions.c \
../../platform/chibios/src/platform_mutex.c\

18
chibios-examples/repl-ChibiOS/main.c
View File

@ -55,8 +55,8 @@ static lbm_cons_t heap[HEAP_SIZE] __attribute__ ((aligned (8)));
static uint32_t memory_array[LBM_MEMORY_SIZE_8K];
static uint32_t bitmap_array[LBM_MEMORY_BITMAP_SIZE_8K];
static lbm_tokenizer_string_state_t string_tok_state;
static lbm_tokenizer_char_stream_t string_tok;
static lbm_string_channel_state_t string_tok_state;
static lbm_char_channel_t string_tok;
BaseSequentialStream *chp = NULL;
@ -465,10 +465,10 @@ int main(void) {
while(lbm_get_eval_state() != EVAL_CPS_STATE_PAUSED) {
sleep_callback(10);
}
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
file_buffer);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
file_buffer);
lbm_cid cid = lbm_load_and_eval_program(&string_tok);
lbm_continue_eval();
@ -489,9 +489,9 @@ int main(void) {
sleep_callback(10);
}
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
str);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
str);
lbm_cid cid = lbm_load_and_eval_expression(&string_tok);

3
chibios-examples/xmas_dac/Makefile
View File

@ -122,7 +122,6 @@ LDSCRIPT= $(STARTUPLD)/STM32F407xG.ld
# C sources that can be compiled in ARM or THUMB mode depending on the global
# setting.
# ../../src/compression.c
LBMSRC = ../../src/env.c \
../../src/eval_cps.c \
../../src/extensions.c \
@ -132,13 +131,13 @@ LBMSRC = ../../src/env.c \
../../src/print.c \
../../src/qq_expand.c \
../../src/stack.c \
../../src/streams.c \
../../src/symrepr.c \
../../src/tokpar.c \
../../src/lispbm.c \
../../src/lbm_c_interop.c \
../../src/lbm_variables.c \
../../src/lbm_custom_type.c \
../../src/lbm_channel.c \
../../platform/chibios/src/platform_mutex.c
CSRC = $(ALLCSRC) \

16
chibios-examples/xmas_dac/main.c
View File

@ -44,8 +44,8 @@ extension_fptr extension_storage[EXTENSION_STORAGE_SIZE];
lbm_cons_t heap[HEAP_SIZE] __attribute__ ((aligned (8)));
static lbm_tokenizer_string_state_t string_tok_state;
static lbm_tokenizer_char_stream_t string_tok;
static lbm_string_channel_state_t string_tok_state;
static lbm_char_channel_t string_tok;
BaseSequentialStream *chp = NULL;
@ -411,9 +411,9 @@ int main(void) {
if (done) {
//lbm_value t;
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
file_buffer);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
file_buffer);
lbm_cid cid = lbm_load_and_eval_program(&string_tok);
lbm_continue_eval();
@ -432,9 +432,9 @@ int main(void) {
sleep_callback(10);
}
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
str);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
str);
lbm_cid cid = lbm_load_and_eval_expression(&string_tok);

78
include/compression.h
View File

@ -1,78 +0,0 @@
/*
Copyright 2019, 2022 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/** \file compression.h */
#ifndef COMPRESSION_H_
#define COMPRESSION_H_
#include <stdint.h>
#include <stdbool.h>
#include "lbm_types.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct {
uint32_t compressed_bits;
uint32_t i;
bool string_mode;
char last_string_char;
char *src;
} decomp_state;
/* This buffer may be too small really... */
#define DECOMP_BUFF_SIZE 32
typedef struct {
decomp_state ds;
char decomp_buff[DECOMP_BUFF_SIZE];
int decomp_bytes;
int buff_pos;
} tokenizer_compressed_state_t;
void lbm_init_compression_state(decomp_state *s, char *src);
/** Compress lisp code. This function is not suitable for execution on a microcontroller.
*
* \param string String to compress. This string cannot be constant.
* The compression algorithm will make destructive updates to the input string.
* \param res_size The resulting, compressed, size is stored in this variable.
* \return A pointer to compressed data allocated using malloc. The user is
* responsible for freeing this memory.
*/
char *lbm_compress(char *string, uint32_t *res_size);
/* Maybe these should not be exposed at all */
int lbm_decompress_incremental(decomp_state *s, char *dest_buff, uint32_t dest_n);
bool lbm_decompress(char *dest, uint32_t dest_n, char *src);
/** Create a tokenizer for compressed data
*
* \param ts Pointer to tokenizer_compressed_state_t to initialize.
* \param str Pointer to tokenizer_char_stream_t to initialize.
* \param bytes Pointer to the compressed data.
*/
void lbm_create_char_stream_from_compressed(tokenizer_compressed_state_t *ts,
lbm_tokenizer_char_stream_t *str,
char *bytes);
#ifdef __cplusplus
}
#endif
#endif

14
include/eval_cps.h
View File

@ -20,6 +20,7 @@
#include "lbm_types.h"
#include "stack.h"
#include "lbm_channel.h"
#ifdef __cplusplus
extern "C" {
@ -245,17 +246,8 @@ void lbm_set_reader_done_callback(void (*fptr)(lbm_cid));
*/
lbm_cid lbm_get_current_cid(void);
/** Create a token stream for parsing for code
*
* \param str character stream to convert into a token stream.
* \return token stream.
*/
lbm_value lbm_create_token_stream(lbm_tokenizer_char_stream_t *str);
/** Explicitly free a stream (if something breaks while creating it)
* The stream must not have been made available to the program
* \param stream The stream to free
*/
int lbm_explicit_free_token_stream(lbm_value stream);
bool create_string_channel(char *str, lbm_value *res);
bool lift_char_channel(lbm_char_channel_t *ch, lbm_value *res);
/** deliver a message
*

16
include/heap.h
View File

@ -23,10 +23,10 @@
#include "lbm_types.h"
#include "symrepr.h"
#include "streams.h"
#include "stack.h"
#include "lbm_memory.h"
#include "lbm_defines.h"
#include "lbm_channel.h"
#ifdef __cplusplus
extern "C" {
@ -310,12 +310,12 @@ lbm_value lbm_heap_allocate_cell(lbm_type type);
* \return String or NULL if the value does not encode a string.
*/
char *lbm_dec_str(lbm_value val);
/** Decode an lbm_value representing a stream into an lbm_stream_t pointer.
/** Decode an lbm_value representing a char channel into an lbm_char_channel_t pointer.
*
* \param val Value
* \return A pointer to an lbm_stream_t or NULL if the value does not encode a stream.
* \return A pointer to an lbm_char_channel_t or NULL if the value does not encode a channel.
*/
lbm_stream_t *lbm_dec_stream(lbm_value val);
lbm_char_channel_t *lbm_dec_channel(lbm_value val);
/** Decode an lbm_value representing a custom type into a lbm_uint value.
*
* \param val Value.
@ -761,13 +761,11 @@ static inline bool lbm_is_array(lbm_value x) {
lbm_type_of(lbm_cdr(x)) == LBM_TYPE_SYMBOL &&
lbm_dec_sym(lbm_cdr(x)) == SYM_ARRAY_TYPE);
}
static inline bool lbm_is_stream(lbm_value x) {
return (lbm_type_of(x) == LBM_TYPE_STREAM &&
static inline bool lbm_is_channel(lbm_value x) {
return (lbm_type_of(x) == LBM_TYPE_CHANNEL &&
lbm_type_of(lbm_cdr(x)) == LBM_TYPE_SYMBOL &&
lbm_dec_sym(lbm_cdr(x)) == SYM_STREAM_TYPE);
lbm_dec_sym(lbm_cdr(x)) == SYM_CHANNEL_TYPE);
}
static inline bool lbm_is_char(lbm_value x) {
lbm_uint t = lbm_type_of(x);
return (t == LBM_TYPE_CHAR);

10
include/lbm_c_interop.h
View File

@ -24,12 +24,12 @@
#include "symrepr.h"
#include "eval_cps.h"
#include "heap.h"
#include "streams.h"
#include "tokpar.h"
#include "lbm_memory.h"
#include "lbm_variables.h"
#include "heap.h"
#include "lbm_types.h"
#include "lbm_channel.h"
#ifdef __cplusplus
extern "C" {
@ -40,27 +40,27 @@ extern "C" {
* \param tokenizer The tokenizer to read the program from.
* \return A context id on success or 0 on failure.
*/
lbm_cid lbm_load_and_eval_program(lbm_tokenizer_char_stream_t *tokenizer);
lbm_cid lbm_load_and_eval_program(lbm_char_channel_t *tokenizer);
/** Load and schedule an expression for execution.
*
* \param tokenizer The tokenizer to read the expression from.
* \return A context id on success or 0 on failure.
*/
lbm_cid lbm_load_and_eval_expression(lbm_tokenizer_char_stream_t *tokenizer);
lbm_cid lbm_load_and_eval_expression(lbm_char_channel_t *tokenizer);
/** Load a program and bind it to a symbol in the environment.
*
* \param tokenizer The tokenizer to read the program from.
* \param symbol A string with the name you want the binding to have in the environment.
* \return A context id on success or 0 on failure.
*/
lbm_cid lbm_load_and_define_program(lbm_tokenizer_char_stream_t *tokenizer, char *symbol);
lbm_cid lbm_load_and_define_program(lbm_char_channel_t *tokenizer, char *symbol);
/** Load an expression and bind it to a symbol in the environment.
*
* \param tokenizer The tokenizer to read the expression from.
* \param symbol A string with the name you want the binding to have in the environment.
* \return A context id on success or 0 on failure.
*/
lbm_cid lbm_load_and_define_expression(lbm_tokenizer_char_stream_t *tokenizer, char *symbol);
lbm_cid lbm_load_and_define_expression(lbm_char_channel_t *tokenizer, char *symbol);
/* Evaluating a definition in a new context */
/** Create a context for a bound expression and schedule it for execution

116
include/lbm_channel.h Normal file
View File

@ -0,0 +1,116 @@
/*
Copyright 2022 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef LBM_CHANNEL_H_
#define LBM_CHANNEL_H_
#include <stdint.h>
#include <stdbool.h>
#include <platform_mutex.h>
#define TOKENIZER_BUFFER_SIZE 257
#define CHANNEL_SUCCESS 1
#define CHANNEL_MORE 2
#define CHANNEL_END 3
#define CHANNEL_FULL 4
#define CHANNEL_READER_CLOSED 1000
typedef struct {
char buffer[TOKENIZER_BUFFER_SIZE];
unsigned int write_pos;
unsigned int read_pos;
bool more;
bool comment;
bool reader_closed;
mutex_t lock;
// statistics
unsigned int row;
unsigned int column;
} lbm_buffered_channel_state_t;
typedef struct {
char *str;
unsigned int read_pos;
bool more;
bool comment;
bool reader_closed;
unsigned int row;
unsigned int column;
} lbm_string_channel_state_t;
typedef struct lbm_char_channel_s {
void *state;
bool (*more)(struct lbm_char_channel_s *ch);
int (*peek)(struct lbm_char_channel_s *ch, unsigned int n, char *res);
bool (*read)(struct lbm_char_channel_s *ch, char *res);
bool (*drop)(struct lbm_char_channel_s *ch, unsigned int n);
bool (*comment)(struct lbm_char_channel_s *ch);
void (*set_comment)(struct lbm_char_channel_s *ch, bool comment);
void (*reader_close)(struct lbm_char_channel_s *ch);
/* Either side */
bool (*channel_is_empty)(struct lbm_char_channel_s *ch);
bool (*channel_is_full)(struct lbm_char_channel_s *ch);
/* Write side */
int (*write)(struct lbm_char_channel_s *ch, char c);
void (*writer_close)(struct lbm_char_channel_s *ch);
/* Statistics */
unsigned int (*row)(struct lbm_char_channel_s *ch);
unsigned int (*column)(struct lbm_char_channel_s *ch);
} lbm_char_channel_t;
/* Read side */
bool lbm_channel_more(lbm_char_channel_t *ch);
int lbm_channel_peek(lbm_char_channel_t *ch, unsigned int n, char *res);
bool lbm_channel_read(lbm_char_channel_t *ch, char *res);
bool lbm_channel_drop(lbm_char_channel_t *ch, unsigned int n);
bool lbm_channel_comment(lbm_char_channel_t *ch);
void lbm_channel_set_comment(lbm_char_channel_t *ch, bool comment);
void lbm_channel_reader_close(lbm_char_channel_t *ch);
/* Either side */
bool lbm_channel_is_empty(lbm_char_channel_t *ch);
bool lbm_channel_is_full(lbm_char_channel_t *ch);
/* Write side */
int lbm_channel_write(lbm_char_channel_t *ch, char c);
void lbm_channel_writer_close(lbm_char_channel_t *ch);
/* Statistics */
unsigned int lbm_channel_row(lbm_char_channel_t *ch);
unsigned int lbm_channel_column(lbm_char_channel_t *ch);
/* Interface */
void lbm_create_string_char_channel(lbm_string_channel_state_t *st,
lbm_char_channel_t *ch,
char *str);
void lbm_create_buffered_char_channel(lbm_buffered_channel_state_t *st,
lbm_char_channel_t *ch);
#endif

14
include/lbm_defines.h
View File

@ -36,7 +36,7 @@
#define LBM_TYPE_DOUBLE 0x70000000u
#define LBM_TYPE_ARRAY 0x80000000u
#define LBM_TYPE_REF 0x90000000u
#define LBM_TYPE_STREAM 0xA0000000u
#define LBM_TYPE_CHANNEL 0xA0000000u
#define LBM_TYPE_CUSTOM 0xB0000000u
#define LBM_NON_CONS_POINTER_TYPE_LAST 0xB0000000u
#define LBM_POINTER_TYPE_LAST 0xB0000000u
@ -64,7 +64,7 @@
#define LBM_TYPE_DOUBLE (lbm_uint)0x4000000000000000
#define LBM_TYPE_ARRAY (lbm_uint)0x5000000000000000
#define LBM_TYPE_REF (lbm_uint)0x6000000000000000
#define LBM_TYPE_STREAM (lbm_uint)0x7000000000000000
#define LBM_TYPE_CHANNEL (lbm_uint)0x7000000000000000
#define LBM_TYPE_CUSTOM (lbm_uint)0x8000000000000000
#define LBM_NON_CONS_POINTER_TYPE_LAST (lbm_uint)0x8000000000000000
#define LBM_POINTER_TYPE_LAST (lbm_uint)0x8000000000000000
@ -133,7 +133,7 @@
#define SYM_IND_I_TYPE 0x34
#define SYM_IND_U_TYPE 0x35
#define SYM_IND_F_TYPE 0x36
#define SYM_STREAM_TYPE 0x37
#define SYM_CHANNEL_TYPE 0x37
#define SYM_BYTECODE_TYPE 0x38
#define SYM_CUSTOM_TYPE 0x39
#define TYPE_CLASSIFIER_ENDS 0x39
@ -166,7 +166,7 @@
#define SYM_TYPE_CHAR 0x5B
#define SYM_TYPE_BYTE 0x5C
#define SYM_TYPE_REF 0x5D
#define SYM_TYPE_STREAM 0x5E
#define SYM_TYPE_CHANNEL 0x5E
//Relevant for the tokenizer
#define SYM_OPENPAR 0x70
@ -251,12 +251,6 @@
#define SYM_TO_DOUBLE 0x167
#define SYM_TO_BYTE 0x168
#define SYM_STREAM_GET 0x169
#define SYM_STREAM_MORE 0x16A
#define SYM_STREAM_PEEK 0x16B
#define SYM_STREAM_DROP 0x16C
#define SYM_STREAM_PUT 0x16D
#define SYM_SHL 0x170
#define SYM_SHR 0x171
#define SYM_BITWISE_AND 0x172

4
include/lbm_types.h
View File

@ -26,7 +26,7 @@
#ifdef __cplusplus
extern "C" {
#endif
/* Addresses that are put into lbm_values or into
* lbm_memory must have this alignment. */
#ifndef LBM64
@ -82,8 +82,6 @@ typedef double lbm_float;
typedef int64_t lbm_cid;
#endif
/* tokenizer */
struct lbm_tcs{

6
include/lbm_version.h
View File

@ -32,8 +32,12 @@ extern "C" {
#define LBM_PATCH_VERSION 0
/*! \page changelog Changelog
Set 16 2022: Version 0.6.0
- Source code can be streamed onto the heap. This means there is no need
for a large buffer on the MCU (or area of flash) to parse source code
from
Set 5 2022: Version 0.6.0
Sep 5 2022: Version 0.6.0
- Refactoring of array-reader. Array reading is nolonger done monolithically
inside of the tokpar framework, but rather as a cooperation between the
evaluator and the tokenizer.

2
include/lispbm.h
View File

@ -28,12 +28,12 @@
#include "tokpar.h"
#include "prelude.h"
#include "env.h"
#include "compression.h"
#include "lbm_memory.h"
#include "lbm_types.h"
#include "lbm_c_interop.h"
#include "lbm_variables.h"
#include "lbm_custom_type.h"
#include "lbm_channel.h"
#ifdef __cplusplus
extern "C" {

96
include/streams.h
View File

@ -1,96 +0,0 @@
/*
Copyright 2021 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/** \file streams.h
* lbm_stream_t implements an abstract stream that can be given many different implementations.
* The operations more, get, peek, drop, put should be implemented for all streams but if they are
* not used or do not make sense for the particular stream they can return for example enc_sym(SYM_NIL).
*
* */
#ifndef STREAMS_H_
#define STREAMS_H_
#include "lbm_types.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct lbm_stream_s{
void *state; /* stream implementation dependent state */
lbm_value (*more)(struct lbm_stream_s*);
lbm_value (*get)(struct lbm_stream_s*);
lbm_value (*peek)(struct lbm_stream_s*, lbm_value);
lbm_value (*drop)(struct lbm_stream_s*, lbm_value);
lbm_value (*put)(struct lbm_stream_s*, lbm_value);
} lbm_stream_t;
/** Get a value from a stream.
*
* \param str Stream to get a value from.
* \return A value.
*/
lbm_value lbm_stream_get(lbm_stream_t *str);
/** Check if there is more data available on the stream.
*
* \param str Stream to query
* \return A stream implementation dependent value indicating if there is more data or not.
*/
lbm_value lbm_stream_more(lbm_stream_t *str);
/** Read the nth value of a stream without removing it from the stream.
*
* \param str Stream to peek into.
* \param n Index to peek at.
* \return Value at that position.
*/
lbm_value lbm_stream_peek(lbm_stream_t *str, lbm_value n);
/** Drop n values from a stream.
*
* \param str Stream to drop values from.
* \param n Number of values to drop.
* \return A stream implementation dependent value.
*/
lbm_value lbm_stream_drop(lbm_stream_t *str, lbm_value n);
/** Put a value onto a stream
*
* \param str Stream to put a value onto.
* \param v Value to put onto the stream.
* \return A Stream implementation dependent value.
*/
lbm_value lbm_stream_put(lbm_stream_t *str, lbm_value v);
/** Create a lispbm value (lbm_value) representing a stream.
*
* \param str Stream to lift into the lisp.
* \return The lbm_value that represents the stream.
*/
lbm_value lbm_stream_create(lbm_stream_t *str);
/** Convert an lbm_tokenizer_char_stream_t to an lbm_stream.
*
* \param char_stream The character stream to lift into an a lispbm stream.
* \return The lispbm stream.
*/
lbm_value lbm_stream_lift(lbm_tokenizer_char_stream_t *char_stream);
#ifdef __cplusplus
}
#endif
#endif

7
include/tokpar.h
View File

@ -20,6 +20,7 @@
#define TOKPAR_H_
#include "lbm_types.h"
#include "lbm_channel.h"
#ifdef __cplusplus
extern "C" {
@ -47,11 +48,13 @@ void lbm_create_char_stream_from_string(lbm_tokenizer_string_state_t *state,
/** Get the next token from a tokenizer stream (lbm_tokenizer_char_stream_t).
*
* \param str Tokenizer stream to get the next token from.
* \param peek Boolean deciding if token is cleared out of channel or left.
* \return an lbm_value representing the token. This token is semi-parsed
* at this stage and values for example are already in proper value form.
*/
lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str);
//lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str);
lbm_value lbm_get_next_token(lbm_char_channel_t *ch, bool peek);
#ifdef __cplusplus
}
#endif

3
lispbm.mk
View File

@ -9,14 +9,13 @@ LISPBM_SRC = $(LISPBM)/src/env.c \
$(LISPBM)/src/stack.c \
$(LISPBM)/src/symrepr.c \
$(LISPBM)/src/tokpar.c \
$(LISPBM)/src/compression.c \
$(LISPBM)/src/extensions.c \
$(LISPBM)/src/lispbm.c \
$(LISPBM)/src/eval_cps.c \
$(LISPBM)/src/streams.c \
$(LISPBM)/src/lbm_c_interop.c \
$(LISPBM)/src/lbm_variables.c \
$(LISPBM)/src/lbm_custom_type.c \
$(LISPBM)/src/lbm_channel.c \
$(LISPBM)/src/extensions/array_extensions.c \
$(LISPBM)/src/extensions/string_extensions.c \
$(LISPBM)/src/extensions/math_extensions.c

236
platform/chibios/src/platform_uart.c
View File

@ -21,7 +21,6 @@
#include "symrepr.h"
#include "heap.h"
#include "extensions.h"
#include "streams.h"
#include <string.h>
#include "platform_chibios_conf.h"
@ -309,240 +308,6 @@ static lbm_value ext_uart_out_avail(lbm_value *args, lbm_uint argn) {
return lbm_enc_i(avail);
}
// Uart low-level character stream interface
typedef struct {
lbm_uart_if_t *uart;
char *buffer;
int read;
int write;
int size;
bool more;
int row;
int column;
} uart_char_stream_state;
static bool buffer_empty(uart_char_stream_state *s) {
return (s->write == s->read);
}
static bool buffer_full(uart_char_stream_state *s) {
return (s->write == s->read - 1 ||
(s->read == 0 && s->write == s->size - 1));
}
static int buffer_elems(uart_char_stream_state *s) {
if (s->write >= s->read) return (s->write - s->read);
return (s->size - (s->read - s->write));
}
static bool buffer_put(uart_char_stream_state *s, char c) {
if (buffer_full(s)) return false;
s->buffer[s->write] = c;
s->write = (s->write + 1) % s->size;
return true;
}
static bool buffer_get(uart_char_stream_state *s, char *c) {
if (s->read == s->write) return false;
*c = s->buffer[s->read];
s->read = (s->read + 1) % s->size;
return true;
}
static bool buffer_peek(uart_char_stream_state *s, int n, char *c) {
if ((s->read + n) % s->size < s->write) {
*c = s->buffer[s->read + n];
return true;
}
return false;
}
static int buffer_up(uart_char_stream_state *s, SerialDriver *drv, int n) {
int num = buffer_elems(s);
if (n < num) return n;
if (n > s->size) return -1;
int to_read = n - num;
int num_read = 0;
msg_t res;
do {
res = sdGetTimeout(drv, TIME_IMMEDIATE);
if (res != MSG_TIMEOUT && res != 0x1A) {
buffer_put(s, res);
num_read ++;
}
} while (num_read < to_read &&
res != MSG_TIMEOUT &&
res != 0x1A);
if (num_read) return num_read + num ;
if (res == MSG_TIMEOUT) return -2;
return 0;
}
static bool uart_stream_more(lbm_tokenizer_char_stream_t *t) {
uart_char_stream_state *state;
state = (uart_char_stream_state*)t->state;
return state->more;
}
static char uart_stream_get(lbm_tokenizer_char_stream_t *t) {
char c;
uart_char_stream_state *s = (uart_char_stream_state *)t->state;
if (!buffer_empty(s)) {
buffer_get(s, &c);
s->column ++;
if (c == '\n') {
s->row++;
s->column = 0;
}
return c;
} else {
SerialDriver *drv = s->uart->drv;
int num_read = 0;
num_read = buffer_up(s, drv, 1);
// This blocking behaviour is unfortunate.
// Doing it the right way may be a big pain though.
if (num_read < 1) {
c = sdGet(drv);
if (c == 0x1A ) s->more = false;
} else {
buffer_get(s, &c);
s->column ++;
if (c == '\n') {
s->row++;
s->column = 0;
}
}
return c;
}
}
static bool uart_stream_put(lbm_tokenizer_char_stream_t *t, char c) {
uart_char_stream_state *state;
state = (uart_char_stream_state*)t->state;
SerialDriver *drv = state->uart->drv;
return false;
}
static char uart_stream_peek(lbm_tokenizer_char_stream_t *t, unsigned int n) {
uart_char_stream_state *s = (uart_char_stream_state *)t->state;
SerialDriver *drv = s->uart->drv;
int num_read = 0;
num_read = buffer_up(s, drv, n);
// This error case should be signaled upwards.
if (num_read < n) {
return '0' + num_read;
} else {
return s->buffer[(s->read + n) % s->size];
}
}
static void uart_stream_drop(lbm_tokenizer_char_stream_t *t, unsigned int n) {
for (unsigned int i = 0; i < n; i ++) {
uart_stream_get(t);
}
}
static unsigned int uart_stream_row(lbm_tokenizer_char_stream_t *t) {
(void)t;
return 0;
}
static unsigned int uart_stream_column(lbm_tokenizer_char_stream_t *t) {
(void) t;
return 0;
}
static lbm_tokenizer_char_stream_t *create_uart_char_stream(lbm_value uart, int buffer_size) {
lbm_tokenizer_char_stream_t *str = NULL;
uart_char_stream_state *state = NULL;
char *buffer = NULL;
buffer = (char *)lbm_memory_allocate(1 + buffer_size / sizeof(lbm_uint));
if (!buffer) {
return NULL;
}
str = (lbm_tokenizer_char_stream_t *)lbm_memory_allocate(1 + sizeof(lbm_tokenizer_char_stream_t) / sizeof(lbm_uint));
if (!str) {
lbm_memory_free((lbm_uint*)buffer);
return NULL;
}
state = (uart_char_stream_state *)lbm_memory_allocate(1 + sizeof(uart_char_stream_state) / sizeof(lbm_uint));
if (!state) {
lbm_memory_free((lbm_uint*)str);
lbm_memory_free((lbm_uint*)buffer);
return NULL;
}
lbm_uart_if_t *lbmu = get_uart_if(uart);
if (!lbmu)
return ENC_SYM_FATAL_ERROR;
state->buffer = buffer;
state->size = buffer_size;
state->write = 0;
state->read = 0;
state->uart = lbmu;
state->more = true; // will remain true until an EOF arrives on stream
state->row = 0;
state->column = 0;
str->state = (void*)state;
str->more = uart_stream_more;
str->get = uart_stream_get;
str->put = uart_stream_put;
str->peek = uart_stream_peek;
str->drop = uart_stream_drop;
str->row = uart_stream_row;
str->column = uart_stream_column;
return str;
}
// Stream extension
static lbm_value ext_uart_stream(lbm_value *args, lbm_uint argn) {
if (argn != 2 || !lbm_is_symbol(args[0]) || !lbm_is_number(args[1])) {
return ENC_SYM_TERROR;
}
int uart = lbm_dec_as_i32(args[0]);
int buffer_size = lbm_dec_as_i32(args[1]);
lbm_tokenizer_char_stream_t *tstr = create_uart_char_stream(uart, buffer_size);
if (tstr == NULL)
return ENC_SYM_MERROR;
return lbm_stream_lift(tstr);
}
// Interface
bool platform_uart_init(void) {
@ -553,7 +318,6 @@ bool platform_uart_init(void) {
res = res && lbm_add_extension("uart-read", ext_uart_read);
res = res && lbm_add_extension("uart-in-avail", ext_uart_in_avail);
res = res && lbm_add_extension("uart-out-avail", ext_uart_out_avail);
res = res && lbm_add_extension("uart-stream", ext_uart_stream);
#ifdef LBM_UART_0
res = res && lbm_str_to_symbol("uart-0", &uart_0_sym);

26
repl/repl.c
View File

@ -32,6 +32,7 @@
#include "extensions/math_extensions.h"
#include "lbm_custom_type.h"
#include "lbm_channel.h"
#define EVAL_CPS_STACK_SIZE 256
#define GC_STACK_SIZE 256
@ -51,8 +52,11 @@ static volatile bool allow_print = true;
struct termios old_termios;
struct termios new_termios;
static lbm_tokenizer_string_state_t string_tok_state;
static lbm_tokenizer_char_stream_t string_tok;
/* static lbm_tokenizer_string_state_t string_tok_state; */
/* static lbm_tokenizer_char_stream_t string_tok; */
static lbm_char_channel_t string_tok;
static lbm_string_channel_state_t string_tok_state;
pthread_mutex_t mut;
@ -649,9 +653,12 @@ int main(int argc, char **argv) {
char *file_str = load_file(&str[5]);
if (file_str) {
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
file_str);
/* lbm_create_char_stream_from_string(&string_tok_state, */
/* &string_tok, */
/* file_str); */
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
file_str);
/* Get exclusive access to the heap */
lbm_pause_eval();
@ -863,9 +870,12 @@ int main(int argc, char **argv) {
sleep_callback(10);
}
//printf("loading: %s\n", str);
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
str);
//lbm_create_char_stream_from_string(&string_tok_state,
// &string_tok,
// str);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
str);
lbm_cid cid = lbm_load_and_eval_expression(&string_tok);
r->str = str;
r->cid = cid;

532
src/compression.c
View File

@ -1,532 +0,0 @@
/*
Copyright 2019, 2021, 2022 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdlib.h>
#include <ctype.h>
#include <lbm_types.h>
#include <string.h>
#include <strings.h>
#include <stdint.h>
#include <stdbool.h>
#include "compression.h"
#include "tokpar.h"
#define KEY 0
#define CODE 1
/* The codes are generated using python script in utils directory
- Depends on the Huffman library (pip3 install huffman)
- exec(open('gen_codes.py').read())
- print(make_c())
- python3 -i gen_codes.py
- print(make_c_codes())
*/
#define NUM_CODES 73
#define MAX_KEY_LENGTH 6
#define MAX_CODE_LENGTH 11
char *codes[NUM_CODES][2] = {
{ "nil", "11110011010" },
{ "cdr", "11110011011" },
{ "car", "11110011100" },
{ "cons", "11110011101" },
{ "let", "11110011110" },
{ "define", "11110011111" },
{ "progn", "1111001000" },
{ "quote", "1111001001" },
{ "list", "1111001010" },
{ "if", "1111001011" },
{ "lambda", "1111001100" },
{ "]", "1101" },
{ "[", "1010" },
{ "((", "1001" },
{ "))", "1011" },
{ ")", "1110" },
{ "(", "1100" },
{ "?", "000111" },
{ "!", "011001" },
{ "z", "011100" },
{ "y", "010001" },
{ "x", "011111" },
{ "w", "010000" },
{ "v", "010100" },
{ "u", "010101" },
{ "t", "001010" },
{ "s", "011101" },
{ "r", "001100" },
{ "q", "100010" },
{ "p", "100001" },
{ "o", "000101" },
{ "n", "001110" },
{ "m", "010010" },
{ "l", "100011" },
{ "k", "001001" },
{ "j", "011010" },
{ "i", "001011" },
{ "h", "011000" },
{ "g", "011011" },
{ "f", "100000" },
{ "e", "010111" },
{ "d", "000110" },
{ "c", "001101" },
{ "b", "010110" },
{ "a", "001111" },
{ "9", "0000111" },
{ "8", "0100110" },
{ "7", "0100111" },
{ "6", "0010000" },
{ "5", "0010001" },
{ "4", "0111100" },
{ "3", "0111101" },
{ "2", "0001000" },
{ "1", "0001001" },
{ "0", "1111000" },
{ "_", "11111111" },
{ ",@", "0000110" },
{ ",", "0000011" },
{ "`", "11110100" },
{ " ", "11110111" },
{ "'", "11111100" },
{ "\\", "11111000" },
{ "\"", "0000100" },
{ "#", "11111110" },
{ ":", "0000101" },
{ ".", "11111010" },
{ ">", "11110101" },
{ "<", "11111101" },
{ "=", "11111001" },
{ "/", "0000000" },
{ "*", "11110110" },
{ "-", "11111011" },
{ "+", "0000010" }
};
int match_longest_key(char *string) {
int longest_match_ix = -1;
unsigned int longest_match_length = 0;
unsigned int n = (unsigned int)strlen(string);
for (int i = 0; i < NUM_CODES; i ++) {
unsigned int s_len = (unsigned int)strlen(codes[i][KEY]);
if (s_len <= n) {
if (strncasecmp(codes[i][KEY], string, s_len) == 0) {
if (s_len > longest_match_length) {
longest_match_ix = i;
longest_match_length = s_len;
}
}
}
}
return longest_match_ix;
}
int match_longest_code(char *string, uint32_t start_bit, uint32_t total_bits) {
uint32_t bits_left = total_bits - start_bit;
int longest_match_ix = -1;
unsigned int longest_match_length = 0;
for (int i = 0; i < NUM_CODES; i++) {
unsigned int s_len = (unsigned int)strlen(codes[i][CODE]);
if ((uint32_t)s_len <= bits_left) {
bool match = true;
for (uint32_t b = 0; b < (uint32_t)s_len; b ++) {
uint32_t byte_ix = (start_bit + b) / 8;
uint32_t bit_ix = (start_bit + b) % 8;
char *code_str = codes[i][CODE];
if (((string[byte_ix] & (1 << bit_ix)) ? '1' : '0') !=
code_str[b]) {
match = false;
}
}
if (match && (s_len > longest_match_length)) {
longest_match_length = s_len;
longest_match_ix = i;
}
}
}
return longest_match_ix;
}
int compressed_length(char *string) {
uint32_t i = 0;
uint32_t n = (uint32_t)strlen(string);
int comp_len = 0; // in bits
bool string_mode = false;
bool gobbling_whitespace = false;
while (i < n) {
if (string_mode) {
if (string[i] == '\"' &&
!(string[i-1] == '\\')) {
string_mode = false;
comp_len += 8;
i++;
} else {
comp_len += 8;
i++;
}
} else {
// Gobble up any comments
if (string[i] == ';' ) {
while (string[i] && string[i] != '\n') {
i++;
}
continue;
}
if ( string[i] == '\n' ||
string[i] == ' ' ||
string[i] == '\t' ||
string[i] == '\r') {
gobbling_whitespace = true;
i ++;
continue;
} else if (gobbling_whitespace) {
gobbling_whitespace = false;
i--;
}
if (string[i] == '\"') string_mode = true;
int ix;
if (isspace(string[i])) {
ix = match_longest_key(" ");
} else {
ix = match_longest_key(string + i);
}
if (ix == -1)return -1;
unsigned int code_len = (unsigned int)strlen(codes[ix][1]);
comp_len += (int)code_len;
i += (unsigned int)strlen(codes[ix][0]);
}
}
return comp_len;
}
void set_bit(char *c, int bit_pos, bool set) {
char bval = 0;
if (bit_pos <= 7) {
bval = (char)(1 << bit_pos);
}
if (set) {
*c = (char)(*c | bval);
} else {
*c = (char)(*c & ~bval);
}
}
void emit_string_char_code(char *compressed, char c, int *bit_pos) {
for (int i = 0; i < 8; i ++) {
int byte_ix = (*bit_pos) / 8;
int bit_ix = (*bit_pos) % 8;
bool s = (c & (1 << i));
set_bit(&compressed[byte_ix], bit_ix, s);
*bit_pos = *bit_pos + 1;
}
}
void emit_code(char *compressed, char *code, int *bit_pos) {
unsigned int n = (unsigned int)strlen(code);
for (unsigned int i = 0; i < n; i ++) {
int byte_ix = (*bit_pos) / 8;
int bit_ix = (*bit_pos) % 8;
bool s = (code[i] == '1');
set_bit(&compressed[byte_ix], bit_ix, s);
*bit_pos = *bit_pos + 1;
}
}
void emit_key(char *dest, char *key, int nk, uint32_t *char_pos) {
for (int i = 0; i < nk; i ++) {
dest[*char_pos] = key[i];
*char_pos = *char_pos + 1;
}
}
char read_character(char *src, uint32_t *bit_pos) {
char c = 0;
for (int i = 0; i < 8; i ++) {
uint32_t byte_ix = (*bit_pos)/8;
uint32_t bit_ix = (*bit_pos)%8;
bool s = src[byte_ix] & (1 << bit_ix);
set_bit(&c, i, s);
*bit_pos = *bit_pos + 1;
}
return c;
}
char *lbm_compress(char *string, uint32_t *res_size) {
int c_size_bits_i = compressed_length(string);
uint32_t c_size_bits = 0;
if ( c_size_bits_i >= 0 ) {
c_size_bits = (uint32_t)compressed_length(string);
}
uint32_t c_size_bytes = 4 + (c_size_bits/8);
if (c_size_bits % 8 > 0) {
c_size_bytes += 1;
}
uint32_t header_value = c_size_bits;
if (header_value == 0) return NULL;
char *compressed = (char*)malloc(c_size_bytes);
if (!compressed) return NULL;
memset(compressed, 0, c_size_bytes);
*res_size = c_size_bytes;
int bit_pos = 0;
compressed[0] = (char)header_value;
compressed[1] = (char)(header_value >> 8);
compressed[2] = (char)(header_value >> 16);
compressed[3] = (char)(header_value >> 24);
bit_pos = 32;
bool string_mode = false;
bool gobbling_whitespace = false;
uint32_t n = (uint32_t) strlen(string);
uint32_t i = 0;
while (i < n) {
if (string_mode) {
if (string[i] == '\"' &&
!(string[i-1] == '\\')) {
emit_string_char_code(compressed, '\"', &bit_pos);
i ++;
string_mode = false;
continue;
} else {
emit_string_char_code(compressed, string[i], &bit_pos);
i++;
}
} else {
// Gobble up any comments
if (string[i] == ';' ) {
while (string[i] && string[i] != '\n') {
i++;
}
continue;
}
// gobble up whitespaces
if ( string[i] == '\n' ||
string[i] == ' ' ||
string[i] == '\t' ||
string[i] == '\r') {
gobbling_whitespace = true;
*(string + i) = ' ';
i ++;
continue;
} else if (gobbling_whitespace) {
gobbling_whitespace = false;
i--;
}
/* Compress string-starting " character */
if (string[i] == '\"') {
string_mode = true;
}
int ix = match_longest_key(&string[i]);
if (ix == -1) {
free(compressed);
return NULL;
}
emit_code(compressed, codes[ix][CODE], &bit_pos);
i += (unsigned int)strlen(codes[ix][0]);
}
}
return compressed;
}
void lbm_init_compression_state(decomp_state *s, char *src) {
memcpy(&s->compressed_bits, src, 4);
s->i = 32;
s->string_mode = false;
s->last_string_char = 0;
s->src = src;
}
int lbm_decompress_incremental(decomp_state *s, char *dest_buff, uint32_t dest_n) {
memset(dest_buff, 0, dest_n);
uint32_t char_pos = 0;
if (s->i < s->compressed_bits + 32) {
if (s->string_mode) {
char c = read_character(s->src, &s->i);
if (c == '\"') {
if (s->last_string_char != '\\') {
s->string_mode = false;
s->last_string_char = 0;
}
}
s->last_string_char = c;
dest_buff[0] = c;
return 1;
}
int ix = match_longest_code(s->src, s->i, (s->compressed_bits + 32));
if (ix == -1) {
return -1;
}
if( strlen(codes[ix][KEY]) == 1 &&
strncmp(codes[ix][KEY], "\"", 1) == 0) {
s->string_mode = true;
s->last_string_char = 0;
}
unsigned int n_bits_decoded = (unsigned int)strlen(codes[ix][CODE]);
emit_key(dest_buff, codes[ix][KEY], (int)strlen(codes[ix][KEY]), &char_pos);
s->i+=n_bits_decoded;
return (int)char_pos;
} else {
return 0;
}
}
bool lbm_decompress(char *dest, uint32_t dest_n, char *src) {
uint32_t char_pos = 0;
char dest_buff[32];
int num_chars = 0;
decomp_state s;
memset(dest, 0, dest_n);
lbm_init_compression_state(&s, src);
while (true) {
num_chars = lbm_decompress_incremental(&s, dest_buff, 32);
if (num_chars == 0) break;
if (num_chars == -1) return false;
for (int i = 0; i < num_chars; i ++) {
dest[char_pos++] = dest_buff[i];
}
}
return true;
}
/* Implementation of the parsing interface */
bool more_compressed(lbm_tokenizer_char_stream_t *str) {
tokenizer_compressed_state_t *s = (tokenizer_compressed_state_t*)str->state;
bool more =
(s->ds.i < s->ds.compressed_bits + 32) ||
(s->buff_pos < s->decomp_bytes);
return more;
}
char get_compressed(lbm_tokenizer_char_stream_t *str) {
tokenizer_compressed_state_t *s = (tokenizer_compressed_state_t*)str->state;
if (s->ds.i >= s->ds.compressed_bits + 32 &&
(s->buff_pos >= s->decomp_bytes)) {
return 0;
}
if (s->buff_pos >= s->decomp_bytes) {
int n = lbm_decompress_incremental(&s->ds, s->decomp_buff,DECOMP_BUFF_SIZE);
if (n == 0) {
return 0;
}
s->decomp_bytes = n;
s->buff_pos = 0;
}
char c = s->decomp_buff[s->buff_pos];
s->buff_pos += 1;
return c;
}
char peek_compressed(lbm_tokenizer_char_stream_t *str, unsigned int n) {
tokenizer_compressed_state_t *s = (tokenizer_compressed_state_t *)str->state;
tokenizer_compressed_state_t old;
memcpy(&old, s, sizeof(tokenizer_compressed_state_t));
char c = get_compressed(str);;
for (unsigned int i = 1; i <= n; i ++) {
c = get_compressed(str);
}
memcpy(str->state, &old, sizeof(tokenizer_compressed_state_t));
return c;
}
void drop_compressed(lbm_tokenizer_char_stream_t *str, unsigned int n) {
for (unsigned int i = 0; i < n; i ++) {
get_compressed(str);
}
}
unsigned int row_compressed(lbm_tokenizer_char_stream_t *str) {
(void) str;
return 0;
}
unsigned int column_compressed(lbm_tokenizer_char_stream_t *str) {
(void) str;
return 0;
}
void lbm_create_char_stream_from_compressed(tokenizer_compressed_state_t *ts,
lbm_tokenizer_char_stream_t *str,
char *bytes) {
ts->decomp_bytes = 0;
memset(ts->decomp_buff, 0, 32);
ts->buff_pos = 0;
lbm_init_compression_state(&ts->ds, bytes);
str->state = ts;
str->more = more_compressed;
str->get = get_compressed;
str->peek = peek_compressed;
str->drop = drop_compressed;
}

304
src/eval_cps.c
View File

@ -25,10 +25,10 @@
#include "fundamental.h"
#include "extensions.h"
#include "exp_kind.h"
#include "streams.h"
#include "tokpar.h"
#include "qq_expand.h"
#include "lbm_variables.h"
#include "lbm_channel.h"
#include "print.h"
#include "platform_mutex.h"
@ -73,7 +73,7 @@
#define READ_TERMINATE_COLON ((41 << LBM_VAL_SHIFT) | LBM_TYPE_U)
#define READ_START_ARRAY ((42 << LBM_VAL_SHIFT) | LBM_TYPE_U)
#define READ_APPEND_ARRAY ((43 << LBM_VAL_SHIFT) | LBM_TYPE_U)
#define READ_CHECK_COLON ((44 << LBM_VAL_SHIFT) | LBM_TYPE_U)
static const char* parse_error_eof = "End of parse stream";
@ -316,121 +316,51 @@ void print_error_message(lbm_value error, unsigned int row, unsigned int col) {
/****************************************************/
/* Tokenizing and parsing */
static lbm_value token_stream_peeker = ENC_SYM_NIL;
static bool token_stream_peeked = false;
bool create_string_channel(char *str, lbm_value *res) {
static lbm_value token_stream_more(lbm_stream_t *str) {
(void) str;
return ENC_SYM_NIL;
}
lbm_char_channel_t *ch = NULL;
lbm_string_channel_state_t *st = NULL;
static lbm_value token_stream_get(lbm_stream_t *str){
if (token_stream_peeked) {
token_stream_peeked = false;
return token_stream_peeker;
st = (lbm_string_channel_state_t*)lbm_memory_allocate(sizeof(lbm_string_channel_state_t) / sizeof(lbm_uint) +1);
if (st == NULL) {
return false;
}
return lbm_get_next_token((lbm_tokenizer_char_stream_t*)str->state);
}
static lbm_value token_stream_peek(lbm_stream_t *str, lbm_value n){
(void)n;
if (token_stream_peeked) {
return token_stream_peeker;
} else {
token_stream_peeker = lbm_get_next_token((lbm_tokenizer_char_stream_t*)str->state);
token_stream_peeked = true;
}
return token_stream_peeker;
}
static lbm_value token_stream_drop(lbm_stream_t *str, lbm_value n){
(void) str;
(void) n;
return ENC_SYM_NIL;
}
static lbm_value token_stream_put(lbm_stream_t *str, lbm_value v){
(void) str;
(void) v;
return ENC_SYM_NIL;
}
lbm_value lbm_create_token_stream(lbm_tokenizer_char_stream_t *str) {
lbm_stream_t *stream;
stream = (lbm_stream_t*)lbm_memory_allocate(1+ sizeof(lbm_stream_t) / (sizeof(lbm_uint)));
if (stream == NULL) {
return ENC_SYM_MERROR;
ch = (lbm_char_channel_t*)lbm_memory_allocate(sizeof(lbm_char_channel_t) / sizeof(lbm_uint) + 1);
if (ch == NULL) {
lbm_memory_free((lbm_uint*)st);
return false;
}
/* gc can check if the state pointer is inside of the
lispbm_memory and free it if that is that case.
Otherwise we assume it is a statically created tokenizer */
stream->state = (void*)str;
stream->more = token_stream_more;
stream->get = token_stream_get;
stream->peek = token_stream_peek;
stream->drop = token_stream_drop;
stream->put = token_stream_put;
lbm_value cell = lbm_heap_allocate_cell(LBM_TYPE_CONS);
if (lbm_type_of(cell) == LBM_TYPE_SYMBOL) {
lbm_memory_free((lbm_uint*)st);
lbm_memory_free((lbm_uint*)ch);
return false;
}
return lbm_stream_create(stream);
lbm_create_string_char_channel(st, ch, str);
lbm_set_car(cell, (lbm_uint)ch);
lbm_set_cdr(cell, lbm_enc_sym(SYM_CHANNEL_TYPE));
cell = lbm_set_ptr_type(cell, LBM_TYPE_CHANNEL);
*res = cell;
return true;
}
int lbm_explicit_free_token_stream(lbm_value stream) {
int r = 0;
if (lbm_is_stream(stream)) {
lbm_stream_t *str = (lbm_stream_t*)lbm_car(stream);
lbm_memory_free((lbm_uint*)str);
stream = lbm_set_ptr_type(stream, LBM_TYPE_CONS);
lbm_set_car(stream, ENC_SYM_NIL);
lbm_set_cdr(stream, ENC_SYM_NIL);
r = 1;
bool lift_char_channel(lbm_char_channel_t *ch, lbm_value *res) {
lbm_value cell = lbm_heap_allocate_cell(LBM_TYPE_CONS);
if (lbm_type_of(cell) == LBM_TYPE_SYMBOL) {
return false;
}
return r;
lbm_set_car(cell, (lbm_uint)ch);
lbm_set_cdr(cell, lbm_enc_sym(SYM_CHANNEL_TYPE));
cell = lbm_set_ptr_type(cell, LBM_TYPE_CHANNEL);
*res = cell;
return true;
}
lbm_value token_stream_from_string_value(lbm_value s) {
char *str = lbm_dec_str(s);
lbm_stream_t *stream = NULL;
lbm_tokenizer_string_state_t *tok_stream_state = NULL;
lbm_tokenizer_char_stream_t *tok_stream = NULL;
stream = (lbm_stream_t*)lbm_memory_allocate(sizeof(lbm_stream_t) / (sizeof(lbm_uint)));
if (stream == NULL) {
return ENC_SYM_MERROR;
}
tok_stream_state = (lbm_tokenizer_string_state_t*)lbm_memory_allocate(1 + (sizeof(lbm_tokenizer_string_state_t) / (sizeof(lbm_uint))));
if (tok_stream_state == NULL) {
lbm_memory_free((lbm_uint*)stream);
return ENC_SYM_MERROR;
}
tok_stream = (lbm_tokenizer_char_stream_t*)lbm_memory_allocate(sizeof(lbm_tokenizer_char_stream_t) / (sizeof(lbm_uint)));
if (tok_stream == NULL) {
lbm_memory_free((lbm_uint*)stream);
lbm_memory_free((lbm_uint*)tok_stream_state);
return ENC_SYM_MERROR;
}
lbm_create_char_stream_from_string(tok_stream_state,
tok_stream,
str);
stream->state = (void*)tok_stream;
stream->more = token_stream_more;
stream->get = token_stream_get;
stream->peek = token_stream_peek;
stream->drop = token_stream_drop;
stream->put = token_stream_put;
return lbm_stream_create(stream);
}
/****************************************************/
@ -1194,49 +1124,16 @@ static inline void dynamic_load(eval_context_t *ctx) {
} else {
CHECK_STACK(lbm_push_2(&ctx->K, ctx->curr_exp, RESUME));
lbm_value cell = lbm_heap_allocate_cell(LBM_TYPE_CONS);
if (lbm_type_of(cell) == LBM_TYPE_SYMBOL) {
gc(ENC_SYM_NIL,ENC_SYM_NIL);
cell = lbm_heap_allocate_cell(LBM_TYPE_CONS);
if (lbm_type_of(cell) == LBM_TYPE_SYMBOL) {
error_ctx(cell);
return;
}
}
lbm_array_header_t *array = (lbm_array_header_t*)lbm_memory_allocate(sizeof(lbm_array_header_t) / (sizeof(lbm_uint)));
if (array == NULL) {
gc(cell,ENC_SYM_NIL);
array = (lbm_array_header_t*)lbm_memory_allocate(sizeof(lbm_array_header_t) / (sizeof(lbm_uint)));
if (array == NULL) {
lbm_value chan;
if (!create_string_channel((char *)code_str, &chan)) {
gc(ENC_SYM_NIL, ENC_SYM_NIL);
if (!create_string_channel((char *)code_str, &chan)) {
error_ctx(ENC_SYM_MERROR);
return;
}
}
array->data = (lbm_uint*)code_str;
array->elt_type = LBM_TYPE_CHAR;
array->size = strlen(code_str);
lbm_set_car(cell, (lbm_uint)array);
lbm_set_cdr(cell, ENC_SYM_ARRAY_TYPE);
cell = lbm_set_ptr_type(cell, LBM_TYPE_ARRAY);
lbm_value stream = token_stream_from_string_value(cell);
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
gc(cell,ENC_SYM_NIL);
stream = token_stream_from_string_value(cell);
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
error_ctx(stream);
return;
}
}
lbm_value loader = ENC_SYM_NIL;
CONS_WITH_GC(loader, stream, loader, stream);
CONS_WITH_GC(loader, chan, loader, chan);
CONS_WITH_GC(loader, ENC_SYM_READ, loader, loader);
lbm_value evaluator = ENC_SYM_NIL;
CONS_WITH_GC(evaluator, loader, evaluator, loader);
@ -1741,29 +1638,24 @@ static inline void apply_setvar(lbm_value *args, lbm_uint nargs, eval_context_t
static inline void apply_read_program(lbm_value *args, lbm_uint nargs, eval_context_t *ctx) {
if (nargs == 1) {
lbm_value stream = ENC_SYM_NIL;
lbm_value chan = ENC_SYM_NIL;
if (lbm_type_of(args[1]) == LBM_TYPE_ARRAY) {
lbm_array_header_t *array = (lbm_array_header_t *)lbm_car(args[1]);
if(array->elt_type == LBM_TYPE_CHAR) {
stream = token_stream_from_string_value(args[1]);
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
gc(ENC_SYM_NIL,ENC_SYM_NIL);
stream = token_stream_from_string_value(args[1]);
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
error_ctx(stream);
return;
}
if (!create_string_channel(lbm_dec_str(args[1]), &chan)) {
gc(ENC_SYM_NIL, ENC_SYM_NIL);
if (!create_string_channel(lbm_dec_str(args[1]), &chan)) {
error_ctx(ENC_SYM_MERROR);
return;
}
}
} else if (lbm_type_of(args[1]) == LBM_TYPE_STREAM) {
stream = args[1];
} else if (lbm_type_of(args[1]) == LBM_TYPE_CHANNEL) {
chan = args[1];
} else {
error_ctx(SYM_EERROR);
return;
}
lbm_value fun = args[0];
lbm_stack_drop(&ctx->K, nargs+1);
CHECK_STACK(lbm_push_3(&ctx->K, stream, fun, READ));
CHECK_STACK(lbm_push_3(&ctx->K, chan, fun, READ));
ctx->r = ENC_SYM_NIL;
ctx->app_cont = true;
}
@ -2293,25 +2185,20 @@ static inline void cont_read(eval_context_t *ctx) {
ctx->app_cont = true;
}
static inline void cont_read_next_token(eval_context_t *ctx) {
static void read_process_token(eval_context_t *ctx, lbm_value stream, lbm_value tok) {
lbm_value stream;
lbm_pop(&ctx->K, &stream);
lbm_stream_t *str = lbm_dec_stream(stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_tokenizer_char_stream_t *s = (lbm_tokenizer_char_stream_t*)str->state;
lbm_value tok = token_stream_get(str);
if (lbm_type_of(tok) == LBM_TYPE_SYMBOL) {
switch (lbm_dec_sym(tok)) {
case SYM_RERROR:
lbm_channel_reader_close(str);
lbm_set_error_reason((char*)parse_error_token);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
done_reading(ctx->id);
return;
case SYM_MERROR:
@ -2320,10 +2207,9 @@ static inline void cont_read_next_token(eval_context_t *ctx) {
return;
case SYM_TOKENIZER_WAIT:
CHECK_STACK(lbm_push_2(&ctx->K, stream, READ_NEXT_TOKEN));
ctx->app_cont = true;
yield_ctx(EVAL_CPS_MIN_SLEEP);
return;
case SYM_TOKENIZER_DONE:
case SYM_TOKENIZER_DONE: {
/* Tokenizer reached "end of file"
The parser could be in a state where it needs
more tokens to correctly finish an expression.
@ -2354,10 +2240,10 @@ static inline void cont_read_next_token(eval_context_t *ctx) {
} else {
/* Parsing failed */
lbm_set_error_reason((char*)parse_error_eof);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
done_reading(ctx->id);
}
break;
} break;
case SYM_CLOSEPAR:
ctx->r = tok;
ctx->app_cont = true;
@ -2400,15 +2286,8 @@ static inline void cont_read_next_token(eval_context_t *ctx) {
ctx->app_cont = true;
break;
default: // arbitrary symbol form
if (token_stream_peek(str, 1) == ENC_SYM_COLON) {
token_stream_get(str); // drop the colon
CHECK_STACK(lbm_push_2(&ctx->K, tok, READ_TERMINATE_COLON));
CHECK_STACK(lbm_push_2(&ctx->K, stream, READ_NEXT_TOKEN));
ctx->app_cont = true;
} else {
ctx->r = tok;
ctx->app_cont = true;
}
CHECK_STACK(lbm_push_3(&ctx->K, stream, tok, READ_CHECK_COLON));
ctx->app_cont = true;
break;
}
} else { // arbitrary value form
@ -2417,6 +2296,22 @@ static inline void cont_read_next_token(eval_context_t *ctx) {
}
}
static inline void cont_read_next_token(eval_context_t *ctx) {
lbm_value stream;
lbm_pop(&ctx->K, &stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_value tok = lbm_get_next_token(str, false);
read_process_token(ctx, stream, tok);
}
static inline void cont_read_start_array(eval_context_t *ctx) {
lbm_value stream;
@ -2477,6 +2372,7 @@ static inline void cont_read_start_array(eval_context_t *ctx) {
CHECK_STACK(lbm_push(&ctx->K, READ_APPEND_ARRAY));
ctx->app_cont = true;
} else {
lbm_channel_reader_close(lbm_dec_channel(stream));
error_ctx(ENC_SYM_RERROR);
}
}
@ -2597,12 +2493,11 @@ static inline void cont_read_expect_closepar(eval_context_t *ctx) {
lbm_pop_2(&ctx->K, &res, &stream);
lbm_stream_t *str = lbm_dec_stream(stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_tokenizer_char_stream_t *s = (lbm_tokenizer_char_stream_t*)str->state;
if (lbm_type_of(ctx->r) == LBM_TYPE_SYMBOL &&
lbm_dec_sym(ctx->r) == SYM_CLOSEPAR) {
@ -2610,7 +2505,7 @@ static inline void cont_read_expect_closepar(eval_context_t *ctx) {
ctx->app_cont = true;
} else {
lbm_set_error_reason((char*)parse_error_close);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
done_reading(ctx->id);
}
}
@ -2627,12 +2522,11 @@ static inline void cont_read_dot_terminate(eval_context_t *ctx) {
lbm_value last_cell = sptr[1];
lbm_value stream = sptr[2];
lbm_stream_t *str = lbm_dec_stream(stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_tokenizer_char_stream_t *s = (lbm_tokenizer_char_stream_t*)str->state;
lbm_stack_drop(&ctx->K ,3);
@ -2640,7 +2534,7 @@ static inline void cont_read_dot_terminate(eval_context_t *ctx) {
(lbm_dec_sym(ctx->r) == SYM_CLOSEPAR ||
lbm_dec_sym(ctx->r) == SYM_DOT)) {
lbm_set_error_reason((char*)parse_error_dot);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
return;
} else {
if (lbm_type_of(last_cell) == LBM_TYPE_CONS) {
@ -2654,7 +2548,7 @@ static inline void cont_read_dot_terminate(eval_context_t *ctx) {
ctx->app_cont = true;
} else {
lbm_set_error_reason((char*)parse_error_dot);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
done_reading(ctx->id);
return;
}
@ -2668,19 +2562,18 @@ static inline void cont_read_done(eval_context_t *ctx) {
lbm_pop_2(&ctx->K, &stream, &prg_tag);
lbm_stream_t *str = lbm_dec_stream(stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_tokenizer_char_stream_t *s = (lbm_tokenizer_char_stream_t*)str->state;
lbm_value tok = token_stream_get(str);
lbm_value tok = lbm_get_next_token(str, false);
/* May not be absolutely required that we check to
see if the tokenizer feels it is done here. */
if (tok != ENC_SYM_TOKENIZER_DONE) {
lbm_set_error_reason((char*)parse_error_eof);
read_error_ctx(s->row(s), s->column(s));
read_error_ctx(lbm_channel_row(str), lbm_channel_column(str));
} else {
ctx->app_cont = true;
}
@ -2726,6 +2619,7 @@ static inline void cont_read_terminate_colon(eval_context_t *ctx) {
lbm_value t2;
lbm_value t3;
lbm_pop(&ctx->K, &sym);
CONS_WITH_GC(t1, ctx->r, term, sym);
CONS_WITH_GC(t2, sym, t1, t1);
CONS_WITH_GC(t3, ENC_SYM_NAMESPACE, t2, t2);
@ -2733,6 +2627,33 @@ static inline void cont_read_terminate_colon(eval_context_t *ctx) {
ctx->app_cont = true;
}
static inline void cont_read_check_colon(eval_context_t *ctx) {
lbm_value r;
lbm_value stream;
lbm_pop_2(&ctx->K, &r, &stream);
lbm_char_channel_t *str = lbm_dec_channel(stream);
if (str == NULL || str->state == NULL) {
error_ctx(ENC_SYM_FATAL_ERROR);
return;
}
lbm_value tok = lbm_get_next_token(str, true);
switch (lbm_dec_sym(tok)) {
case SYM_COLON:
lbm_get_next_token(str,false); //drop the colon;
CHECK_STACK(lbm_push_2(&ctx->K, r, READ_TERMINATE_COLON));
CHECK_STACK(lbm_push_2(&ctx->K, stream, READ_NEXT_TOKEN));
ctx->app_cont = true;
break;
default:
ctx->r = r;
ctx->app_cont = true;
}
}
static inline void cont_application_start(eval_context_t *ctx) {
@ -2884,6 +2805,7 @@ static void evaluation_step(void){
case READ_TERMINATE_COLON: cont_read_terminate_colon(ctx); return;
case READ_START_ARRAY: cont_read_start_array(ctx); return;
case READ_APPEND_ARRAY: cont_read_append_array(ctx); return;
case READ_CHECK_COLON: cont_read_check_colon(ctx); return;
default:
error_ctx(ENC_SYM_EERROR);
return;
@ -2926,7 +2848,7 @@ static void evaluation_step(void){
case LBM_TYPE_CHAR:
case LBM_TYPE_ARRAY:
case LBM_TYPE_REF:
case LBM_TYPE_STREAM: eval_selfevaluating(ctx); return;
case LBM_TYPE_CHANNEL: eval_selfevaluating(ctx); return;
case LBM_TYPE_CONS: {
head = lbm_car(ctx->curr_exp);

26
src/fundamental.c
View File

@ -1323,32 +1323,6 @@ lbm_value lbm_fundamental(lbm_value* args, lbm_uint nargs, lbm_value op) {
result = bitwise_not(args[0]);
}
break;
case SYM_STREAM_GET:
if (lbm_type_of(args[0]) == LBM_TYPE_STREAM) {
lbm_stream_t *str = lbm_dec_stream(args[0]);
return str->get(str);
}
break;
case SYM_STREAM_PUT:
if (nargs == 2 && lbm_type_of(args[0]) == LBM_TYPE_STREAM) {
lbm_stream_t *str = lbm_dec_stream(args[0]);
return str->put(str, args[1]);
}
break;
case SYM_STREAM_PEEK:
if (nargs == 2 &&
lbm_type_of(args[0]) == LBM_TYPE_STREAM) {
lbm_stream_t *str = lbm_dec_stream(args[0]);
return str->peek(str, args[1]);
}
break;
case SYM_STREAM_DROP:
if (nargs == 2 &&
lbm_type_of(args[0]) == LBM_TYPE_STREAM) {
lbm_stream_t *str = lbm_dec_stream(args[0]);
return str->drop(str, args[1]);
}
break;
case SYM_CUSTOM_DESTRUCT:
if (nargs == 1 && (lbm_type_of(args[0]) == LBM_TYPE_CUSTOM)) {
lbm_uint *mem_ptr = (lbm_uint*)lbm_dec_custom(args[0]);

19
src/heap.c
View File

@ -26,6 +26,7 @@
#include "heap.h"
#include "symrepr.h"
#include "stack.h"
#include "lbm_channel.h"
#ifdef VISUALIZE_HEAP
#include "heap_vis.h"
#endif
@ -47,16 +48,14 @@ char *lbm_dec_str(lbm_value val) {
}
return res;
}
lbm_char_channel_t *lbm_dec_channel(lbm_value val) {
lbm_char_channel_t *res = NULL;
lbm_stream_t *lbm_dec_stream(lbm_value val) {
lbm_stream_t *res = 0;
if (lbm_type_of(val) == LBM_TYPE_STREAM) {
res = (lbm_stream_t *)lbm_car(val);
if (lbm_type_of(val) == LBM_TYPE_CHANNEL) {
res = (lbm_char_channel_t *)lbm_car(val);
}
return res;
}
lbm_uint lbm_dec_custom(lbm_value val) {
lbm_uint res = 0;
if (lbm_type_of(val) == LBM_TYPE_CUSTOM) {
@ -514,10 +513,10 @@ int lbm_gc_sweep_phase(void) {
}
lbm_memory_free((lbm_uint *)arr);
} break;
case SYM_STREAM_TYPE:{
lbm_stream_t *stream = (lbm_stream_t*)heap[i].car;
if (lbm_memory_ptr_inside((lbm_uint*)stream)) {
lbm_memory_free((lbm_uint*)stream);
case SYM_CHANNEL_TYPE:{
lbm_char_channel_t *chan = (lbm_char_channel_t*)heap[i].car;
if (lbm_memory_ptr_inside((lbm_uint*)chan)) {
lbm_memory_free((lbm_uint*)chan);
}
} break;
case SYM_CUSTOM_TYPE: {

42
src/lbm_c_interop.c
View File

@ -21,9 +21,13 @@
/* Interface for loading and running programs and */
/* expressions */
lbm_cid eval_cps_load_and_eval(lbm_tokenizer_char_stream_t *tokenizer, bool program) {
lbm_cid eval_cps_load_and_eval(lbm_char_channel_t *tokenizer, bool program) {
lbm_value stream = lbm_create_token_stream(tokenizer);
lbm_value stream;
if (!lift_char_channel(tokenizer, &stream)) {
return 0;
}
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
// TODO: Check what should be done.
@ -43,15 +47,19 @@ lbm_cid eval_cps_load_and_eval(lbm_tokenizer_char_stream_t *tokenizer, bool prog
if (lbm_type_of(launcher) != LBM_TYPE_CONS ||
lbm_type_of(evaluator) != LBM_TYPE_CONS ||
lbm_type_of(start_prg) != LBM_TYPE_CONS ) {
lbm_explicit_free_token_stream(stream);
//lbm_explicit_free_token_stream(stream);
return 0;
}
return lbm_create_ctx(start_prg, lbm_enc_sym(SYM_NIL), 256);
}
lbm_cid eval_cps_load_and_define(lbm_tokenizer_char_stream_t *tokenizer, char *symbol, bool program) {
lbm_cid eval_cps_load_and_define(lbm_char_channel_t *tokenizer, char *symbol, bool program) {
lbm_value stream = lbm_create_token_stream(tokenizer);
lbm_value stream;
if (!lift_char_channel(tokenizer, &stream)) {
return 0;
}
if (lbm_type_of(stream) == LBM_TYPE_SYMBOL) {
return 0;
@ -61,7 +69,7 @@ lbm_cid eval_cps_load_and_define(lbm_tokenizer_char_stream_t *tokenizer, char *s
if (!lbm_get_symbol_by_name(symbol, &sym_id)) {
if (!lbm_add_symbol(symbol, &sym_id)) {
lbm_explicit_free_token_stream(stream);
//lbm_explicit_free_token_stream(stream);
return 0;
}
}
@ -79,7 +87,7 @@ lbm_cid eval_cps_load_and_define(lbm_tokenizer_char_stream_t *tokenizer, char *s
if (lbm_type_of(launcher) != LBM_TYPE_CONS ||
lbm_type_of(binding) != LBM_TYPE_CONS ||
lbm_type_of(definer) != LBM_TYPE_CONS ) {
lbm_explicit_free_token_stream(stream);
//lbm_explicit_free_token_stream(stream);
return 0;
}
return lbm_create_ctx(definer, lbm_enc_sym(SYM_NIL), 256);
@ -120,19 +128,19 @@ lbm_cid lbm_eval_defined(char *symbol, bool program) {
lbm_cid lbm_load_and_eval_expression(lbm_tokenizer_char_stream_t *tokenizer) {
lbm_cid lbm_load_and_eval_expression(lbm_char_channel_t *tokenizer) {
return eval_cps_load_and_eval(tokenizer, false);
}
lbm_cid lbm_load_and_define_expression(lbm_tokenizer_char_stream_t *tokenizer, char *symbol) {
lbm_cid lbm_load_and_define_expression(lbm_char_channel_t *tokenizer, char *symbol) {
return eval_cps_load_and_define(tokenizer, symbol, false);
}
lbm_cid lbm_load_and_eval_program(lbm_tokenizer_char_stream_t *tokenizer) {
lbm_cid lbm_load_and_eval_program(lbm_char_channel_t *tokenizer) {
return eval_cps_load_and_eval(tokenizer, true);
}
lbm_cid lbm_load_and_define_program(lbm_tokenizer_char_stream_t *tokenizer, char *symbol) {
lbm_cid lbm_load_and_define_program(lbm_char_channel_t *tokenizer, char *symbol) {
return eval_cps_load_and_define(tokenizer, symbol, true);
}
@ -167,16 +175,16 @@ int lbm_define(char *symbol, lbm_value value) {
if (strncmp(symbol, "#",1) == 0) {
if (!lbm_get_symbol_by_name(symbol, &sym_id)) {
if (!lbm_add_variable_symbol_const(symbol, &sym_id)) {
return 0;
}
if (!lbm_add_variable_symbol_const(symbol, &sym_id)) {
return 0;
}
}
lbm_set_var(sym_id, value);
} else {
if (!lbm_get_symbol_by_name(symbol, &sym_id)) {
if (!lbm_add_symbol_const(symbol, &sym_id)) {
return 0;
}
if (!lbm_add_symbol_const(symbol, &sym_id)) {
return 0;
}
}
*lbm_get_env_ptr() = lbm_env_set(lbm_get_env(), lbm_enc_sym(sym_id), value);
}

359
src/lbm_channel.c Normal file
View File

@ -0,0 +1,359 @@
/*
Copyright 2022 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <lbm_channel.h>
#include <string.h>
/* ------------------------------------------------------------
Interface
------------------------------------------------------------ */
bool lbm_channel_more(lbm_char_channel_t *ch) {
return ch->more(ch);
}
int lbm_channel_peek(lbm_char_channel_t *ch, unsigned int n, char *res) {
return ch->peek(ch, n, res);
}
bool lbm_channel_read(lbm_char_channel_t *ch, char *res) {
return ch->read(ch, res);
}
bool lbm_channel_drop(lbm_char_channel_t *ch, unsigned int n) {
return ch->drop(ch, n);
}
bool lbm_channel_comment(lbm_char_channel_t *ch) {
return ch->comment(ch);
}
void lbm_channel_set_comment(lbm_char_channel_t *ch, bool comment) {
ch->set_comment(ch, comment);
}
bool lbm_channel_is_empty(lbm_char_channel_t *ch) {
return ch->channel_is_empty(ch);
}
bool lbm_channel_is_full(lbm_char_channel_t *ch) {
return ch->channel_is_full(ch);
}
int lbm_channel_write(lbm_char_channel_t *ch, char c) {
return ch->write(ch, c);
}
void lbm_channel_writer_close(lbm_char_channel_t *ch) {
ch->writer_close(ch);
}
void lbm_channel_reader_close(lbm_char_channel_t *ch) {
ch->reader_close(ch);
}
unsigned int lbm_channel_row(lbm_char_channel_t *ch) {
return ch->row(ch);
}
unsigned int lbm_channel_column(lbm_char_channel_t *ch) {
return ch->column(ch);
}
/* ------------------------------------------------------------
Implementation buffered channel
------------------------------------------------------------ */
bool buffered_more(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
return st->more;
}
void buffered_writer_close(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
st->more = false;
}
void buffered_reader_close(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
st->reader_closed = true;
}
int buffered_peek(lbm_char_channel_t *ch, unsigned int n, char *res) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
char *buffer = st->buffer;
int ret = CHANNEL_MORE;
mutex_lock(&st->lock);
unsigned int peek_pos = (st->read_pos + n) % TOKENIZER_BUFFER_SIZE;
bool in_data;
if (st->write_pos >= st->read_pos) {
in_data = peek_pos < st->write_pos && peek_pos >= st->read_pos;
} else {
in_data = !(peek_pos >= st->write_pos && peek_pos < st->read_pos);
}
if (in_data) {
*res = buffer[peek_pos];
ret = CHANNEL_SUCCESS;
} else if (!buffered_more(ch)) {
ret = CHANNEL_END;
} else if (buffered_more(ch)) {
ret = CHANNEL_MORE;
}
mutex_unlock(&st->lock);
return ret;
}
bool buffered_channel_is_empty(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
if (st->read_pos == st->write_pos) {
return true;
}
return false;
}
bool buffered_channel_is_full(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
if (st->write_pos == st->read_pos - 1 ||
(st->read_pos == 0 &&
st->write_pos == TOKENIZER_BUFFER_SIZE-1)) {
return true;
}
return false;
}
bool buffered_read(lbm_char_channel_t *ch, char *res) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
char *buffer = st->buffer;
bool ret = false;
mutex_lock(&st->lock);
if (!buffered_channel_is_empty(ch)) {
*res = buffer[st->read_pos];
st->read_pos = (st->read_pos + 1) % TOKENIZER_BUFFER_SIZE;
ret = true;
}
mutex_unlock(&st->lock);
return ret;
}
bool buffered_drop(lbm_char_channel_t *ch, unsigned int n) {
bool r = true;
char c;
for (unsigned int i = 0; i < n; i ++) {
r = buffered_read(ch, &c);
if (r == false) break;
}
return r;
}
int buffered_write(lbm_char_channel_t *ch, char c) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
if (st->reader_closed) return CHANNEL_READER_CLOSED;
int ret = CHANNEL_FULL;
mutex_lock(&st->lock);
char *buffer = st->buffer;
if (!buffered_channel_is_full(ch)) {
buffer[st->write_pos] = c;
st->write_pos = (st->write_pos + 1) % TOKENIZER_BUFFER_SIZE;
ret = CHANNEL_SUCCESS;
}
mutex_unlock(&st->lock);
return ret;
}
unsigned int buffered_row(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
return st->row;
}
unsigned int buffered_column(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
return st->column;
}
bool buffered_comment(lbm_char_channel_t *ch) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
return st->comment;
}
void buffered_set_comment(lbm_char_channel_t *ch, bool comment) {
lbm_buffered_channel_state_t *st = (lbm_buffered_channel_state_t*)ch->state;
st->comment = comment;
}
void lbm_create_buffered_char_channel(lbm_buffered_channel_state_t *st,
lbm_char_channel_t *ch) {
st->write_pos = 0;
st->read_pos = 0;
st->more = true;
st->reader_closed = false;
st->comment = false;
st->row = 0;
st->column = 0;
mutex_init(&st->lock);
ch->state = st;
ch->more = buffered_more;
ch->peek = buffered_peek;
ch->read = buffered_read;
ch->drop = buffered_drop;
ch->comment = buffered_comment;
ch->set_comment = buffered_set_comment;
ch->channel_is_empty = buffered_channel_is_empty;
ch->channel_is_full = buffered_channel_is_full;
ch->write = buffered_write;
ch->writer_close = buffered_writer_close;
ch->reader_close = buffered_reader_close;
ch->row = buffered_row;
ch->column = buffered_column;
}
/* ------------------------------------------------------------
Implementation string channel
------------------------------------------------------------ */
bool string_more(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
return st->more;
}
void string_writer_close(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
st->more = false;
}
void string_reader_close(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
st->reader_closed = true;
}
int string_peek(lbm_char_channel_t *ch, unsigned int n, char *res) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
char *str = st->str;
unsigned int peek_pos = st->read_pos + n;
if (peek_pos < strlen(str)) {
*res = str[peek_pos];
return CHANNEL_SUCCESS;
}
return CHANNEL_END;
}
bool string_channel_is_empty(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
if (st->read_pos == strlen(st->str)) {
return true;
}
return false;
}
bool string_channel_is_full(lbm_char_channel_t *ch) {
(void)ch;
return true;
}
bool string_read(lbm_char_channel_t *ch, char *res) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
char *str = st->str;
if (st->read_pos < strlen(str)) {
*res = str[st->read_pos];
if (*res == '\n') {
st->row ++;
st->column = 1;
} else if (*res == 0) {
st->more = false;
} else {
st->column++;
}
st->read_pos = st->read_pos + 1;
} else {
st->more = false;
}
return true;
}
bool string_drop(lbm_char_channel_t *ch, unsigned int n) {
bool r = true;
char c;
if (n > 0) {
do {
r = string_read(ch, &c);
n--;
} while (n > 0 && r);
}
return r;
}
int string_write(lbm_char_channel_t *ch, char c) {
(void) ch;
(void) c;
return CHANNEL_FULL;
}
unsigned int string_row(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
return st->row;
}
unsigned int string_column(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
return st->column;
}
bool string_comment(lbm_char_channel_t *ch) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
return st->comment;
}
void string_set_comment(lbm_char_channel_t *ch, bool comment) {
lbm_string_channel_state_t *st = (lbm_string_channel_state_t*)ch->state;
st->comment = comment;
}
void lbm_create_string_char_channel(lbm_string_channel_state_t *st,
lbm_char_channel_t *ch,
char *str) {
st->str = str;
st->read_pos = 0;
st->more = false;
st->comment = false;
st->row = 0;
st->column = 0;
ch->state = st;
ch->more = string_more;
ch->peek = string_peek;
ch->read = string_read;
ch->drop = string_drop;
ch->comment = string_comment;
ch->set_comment = string_set_comment;
ch->channel_is_empty = string_channel_is_empty;
ch->channel_is_full = string_channel_is_full;
ch->write = string_write;
ch->writer_close = string_writer_close;
ch->reader_close = string_reader_close;
ch->row = string_row;
ch->column = string_column;
}

4
src/print.c
View File

@ -339,9 +339,9 @@ int lbm_print_value(char *buf,unsigned int len, lbm_value t) {
}
break;
}
case LBM_TYPE_STREAM: {
case LBM_TYPE_CHANNEL: {
r = snprintf(buf + offset, len - offset, "~STREAM~");
r = snprintf(buf + offset, len - offset, "~CHANNEL~");
if ( r > 0) {
n = (unsigned int) r;
} else {

98
src/streams.c
View File

@ -1,98 +0,0 @@
/*
Copyright 2021, 2022 Joel Svensson svenssonjoel@yahoo.se
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "streams.h"
#include "heap.h"
#include "lbm_types.h"
#include "lbm_memory.h"
#include "symrepr.h"
lbm_value lbm_stream_get(lbm_stream_t *str) {
return str->get(str);
}
lbm_value lbm_stream_more(lbm_stream_t *str) {
return str->more(str);
}
lbm_value lbm_stream_peek(lbm_stream_t *str, lbm_value n) {
return str->peek(str,n);
}
lbm_value lbm_stream_drop(lbm_stream_t *str, lbm_value n) {
return str->drop(str,n);
}
lbm_value lbm_stream_put(lbm_stream_t *str, lbm_value v) {
return str->put(str,v);
}
lbm_value lbm_stream_create(lbm_stream_t *str) {
lbm_value s = lbm_cons((lbm_value)str, lbm_enc_sym(SYM_STREAM_TYPE));
if (lbm_type_of(s) == LBM_TYPE_CONS) {
s = lbm_set_ptr_type(s, LBM_TYPE_STREAM);
}
return s;
}
// Lift low-level character streams to lisp stream.
static lbm_value lift_get(lbm_stream_t *str) {
lbm_tokenizer_char_stream_t *cs = (lbm_tokenizer_char_stream_t*)str->state;
return lbm_enc_char(cs->get(cs));
}
static lbm_value lift_put(lbm_stream_t *str, lbm_value v) {
char c = lbm_dec_as_char(v);
lbm_tokenizer_char_stream_t *cs = (lbm_tokenizer_char_stream_t*)str->state;
return cs->put(cs, c) ? ENC_SYM_TRUE : ENC_SYM_NIL;
}
static lbm_value lift_more(lbm_stream_t *str) {
lbm_tokenizer_char_stream_t *cs = (lbm_tokenizer_char_stream_t*)str->state;
return cs->get(cs) ? ENC_SYM_TRUE : ENC_SYM_NIL;
}
static lbm_value lift_peek(lbm_stream_t *str, lbm_value n) {
if (!lbm_is_number(n)) return ENC_SYM_TERROR;
lbm_tokenizer_char_stream_t *cs = (lbm_tokenizer_char_stream_t*)str->state;
return lbm_enc_char(cs->peek(cs, lbm_dec_as_u32(n)));
}
static lbm_value lift_drop(lbm_stream_t *str, lbm_value n) {
if (!lbm_is_number(n)) return ENC_SYM_TERROR;
lbm_tokenizer_char_stream_t *cs = (lbm_tokenizer_char_stream_t*)str->state;
cs->drop(cs, lbm_dec_as_u32(n));
return ENC_SYM_TRUE;
}
lbm_value lbm_stream_lift(lbm_tokenizer_char_stream_t *char_stream) {
lbm_stream_t *stream = NULL;
stream = (lbm_stream_t *)lbm_memory_allocate(1 + sizeof(lbm_stream_t) / sizeof(lbm_uint));
if (stream == NULL) {
return ENC_SYM_MERROR;
}
stream->state = (void*)char_stream;
stream->more = lift_more;
stream->get = lift_get;
stream->put = lift_put;
stream->peek = lift_peek;
stream->drop = lift_drop;
return lbm_stream_create(stream);
}

11
src/symrepr.c
View File

@ -89,7 +89,7 @@ special_sym const special_symbols[] = {
{"sym_ind_i" , SYM_IND_I_TYPE},
{"sym_ind_u" , SYM_IND_U_TYPE},
{"sym_ind_f" , SYM_IND_F_TYPE},
{"sym_stream" , SYM_STREAM_TYPE},
{"sym_channel" , SYM_CHANNEL_TYPE},
{"sym_recovered" , SYM_RECOVERED},
{"sym_bytecode" , SYM_BYTECODE_TYPE},
{"sym_custom" , SYM_CUSTOM_TYPE},
@ -125,7 +125,7 @@ special_sym const special_symbols[] = {
{"type-char" , SYM_TYPE_CHAR},
{"type-byte" , SYM_TYPE_BYTE},
{"type-ref" , SYM_TYPE_REF},
{"type-stream" , SYM_TYPE_STREAM},
{"type-channel" , SYM_TYPE_CHANNEL},
// Fundamental operations
{"+" , SYM_ADD},
{"-" , SYM_SUB},
@ -190,13 +190,6 @@ special_sym const special_symbols[] = {
{"to-double" , SYM_TO_DOUBLE},
{"to-byte" , SYM_TO_BYTE},
// Streams
{"stream-get" , SYM_STREAM_GET},
{"stream-more" , SYM_STREAM_MORE},
{"stream-peek" , SYM_STREAM_PEEK},
{"stream-drop" , SYM_STREAM_DROP},
{"stream-put" , SYM_STREAM_PUT},
// fast access in list
{"ix" , SYM_IX},

755
src/tokpar.c
View File

@ -16,17 +16,17 @@
*/
#include <stdbool.h>
#include <stdio.h>
//#include <stdio.h>
#include <ctype.h>
#include <lbm_memory.h>
#include <lbm_types.h>
#include <string.h>
#include <stdlib.h>
#include "lbm_memory.h"
#include "lbm_types.h"
#include "lbm_channel.h"
#include "tokpar.h"
#include "symrepr.h"
#include "heap.h"
#include "compression.h"
#include "qq_expand.h"
#include "env.h"
@ -65,48 +65,60 @@
#define TOKCOLON 32u
#define TOKTYPEBYTE 34u
#define TOKTYPEI 35u
#define TOKTYPEU 36u
#define TOKTYPEI32 37u
#define TOKTYPEU32 38u
#define TOKTYPEI64 39u
#define TOKTYPEU64 40u
#define TOKTYPEF32 41u
#define TOKTYPEF64 42u
#define TOKNEEDMORE 1000u
#define TOKENIZER_ERROR 1024u
#define TOKENIZER_END 2048u
#define TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH 256
// Tokenizer return values
// > 0 : successfully found token
// = 0 : tokenizer can definitely not create a token
// < 0 : tokenizer does not know if it can or cannot create a token yet.
#define TOKENIZER_NO_TOKEN 0
#define TOKENIZER_NEED_MORE -1
#define TOKENIZER_STRING_ERROR -2
static char sym_str[TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH];
static void clear_sym_str(void) {
memset(sym_str,0,TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH);
}
#define TOK_TYPE_NONE 0
#define TOK_TYPE_BYTE 1
#define TOK_TYPE_I 2
#define TOK_TYPE_U 3
#define TOK_TYPE_I32 4
#define TOK_TYPE_U32 5
#define TOK_TYPE_I64 6
#define TOK_TYPE_U64 7
#define TOK_TYPE_FLOAT 8
#define TOK_TYPE_DOUBLE 9
typedef struct {
int type;
uint32_t type;
uint64_t value;
bool negative;
} token_int;
typedef struct token_float {
int type;
uint32_t type;
double value;
bool negative;
} token_float;
typedef struct {
const char *str;
uint32_t token;
uint32_t token;
uint32_t len;
} matcher;
#define NUM_FIXED_SIZE_TOKENS 21
const matcher match_table[NUM_FIXED_SIZE_TOKENS] = {
const matcher fixed_size_tokens[NUM_FIXED_SIZE_TOKENS] = {
{"(", TOKOPENPAR, 1},
{")", TOKCLOSEPAR, 1},
{"[", TOKOPENBRACK, 1},
@ -130,40 +142,46 @@ const matcher match_table[NUM_FIXED_SIZE_TOKENS] = {
{"?", TOKMATCHANY, 1}
};
bool more(lbm_tokenizer_char_stream_t *str) {
return str->more(str);
}
#define NUM_TYPE_QUALIFIERS 9
const matcher type_qual_table[NUM_TYPE_QUALIFIERS] = {
{"f64", TOKTYPEF64, 3},
{"f32", TOKTYPEF32, 3},
{"i64", TOKTYPEI64, 3},
{"u64", TOKTYPEU64, 3},
{"i32", TOKTYPEI32, 3},
{"u32", TOKTYPEU32, 3},
{"i" , TOKTYPEI, 1},
{"u" , TOKTYPEU, 1},
{"b" , TOKTYPEBYTE, 1}
};
char get(lbm_tokenizer_char_stream_t *str) {
return str->get(str);
}
int tok_match_fixed_size_tokens(lbm_char_channel_t *ch, const matcher *m, unsigned int start_pos, unsigned int num, uint32_t *res) {
char peek(lbm_tokenizer_char_stream_t *str, unsigned int n) {
return str->peek(str,n);
}
void drop(lbm_tokenizer_char_stream_t *str, unsigned int n) {
str->drop(str,n);
}
uint32_t tok_match_fixed_size_tokens(lbm_tokenizer_char_stream_t *str) {
for (int i = 0; i < NUM_FIXED_SIZE_TOKENS; i ++) {
uint32_t tok_len = match_table[i].len;
const char *match_str = match_table[i].str;
uint32_t tok = match_table[i].token;
uint32_t char_pos;
for (char_pos = 0; char_pos < tok_len; char_pos ++) {
if (peek(str,char_pos) != match_str[char_pos]) break;
for (unsigned int i = 0; i < num; i ++) {
uint32_t tok_len = m[i].len;
const char *match_str = m[i].str;
uint32_t tok = m[i].token;
char c;
int char_pos;
int r;
for (char_pos = 0; char_pos < (int)tok_len; char_pos ++) {
r = lbm_channel_peek(ch,(unsigned int)char_pos + start_pos, &c);
if (r == CHANNEL_SUCCESS) {
if (c != match_str[char_pos]) break;
} else if (r == CHANNEL_MORE ) {
*res = TOKNEEDMORE;
return TOKENIZER_NEED_MORE;
} else {
break;
}
}
if (char_pos == tok_len) { //match
drop(str,tok_len);
return tok;
if (char_pos == (int)tok_len) { //match
*res = tok;
return (int)tok_len;
}
}
return NOTOKEN;
return TOKENIZER_NO_TOKEN;
}
bool symchar0(char c) {
@ -186,35 +204,30 @@ bool symchar(char c) {
return false;
}
int tok_symbol(lbm_tokenizer_char_stream_t *str) {
int tok_symbol(lbm_char_channel_t *ch) {
if (!symchar0(peek(str,0))) return 0;
char c;
unsigned int i = 0;
unsigned int len = 1;
int n = 0;
while (symchar((peek(str,len)))) {
len++;
if (lbm_channel_peek(ch, 0, &c) != CHANNEL_SUCCESS || !symchar0(c)) {
return TOKENIZER_NO_TOKEN;
}
if (len > TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH)
return -1; /* TODO: specific error code that can be presented to user */
clear_sym_str();
sym_str[0] = (char)tolower(c);
int c = 0;
int len = 1;
int r = 0;
for (i = 0; i < len; i ++) {
c = tolower(get(str));
if (c >= 0 && c <= 255) {
sym_str[i] = (char)c;
n++;
} else {
return -1;
r = lbm_channel_peek(ch,(unsigned int)len, &c);
while (r == CHANNEL_SUCCESS && symchar(c)) {
c = (char)tolower(c);
if (len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
sym_str[len] = (char)c;
}
len ++;
r = lbm_channel_peek(ch,(unsigned int)len, &c);
}
return (int)n;
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
return len;
}
static char translate_escape_char(char c) {
@ -228,251 +241,365 @@ static char translate_escape_char(char c) {
}
}
int tok_string(lbm_tokenizer_char_stream_t *str) {
unsigned int i = 0;
int n = 0;
unsigned int len = 0;
if (!(peek(str,0) == '\"')) return 0;
get(str); // remove the " char
n++;
// compute length of string
char c;
do {
c = peek(str,len);
if (c == '\\') {
len +=2;
} else {
len ++;
}
} while (c != 0 &&
c != '\"');
len = len -1;
if (len > TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH)
return -1; /* TODO: specific error code that can be presented to user */
// str ends before tokenized string is closed.
if ((peek(str,len)) != '\"') {
return 0;
}
clear_sym_str();
for (i = 0; i < len; i ++) {
c = get(str);
if (c == '\\') {
if (i + 1 < len) {
char escaped = get(str);
c = translate_escape_char(escaped);
len-=1;
}
}
sym_str[i] = c;
n++;
}
get(str); // throw away the "
return (int)(n+1);
}
int tok_char(lbm_tokenizer_char_stream_t *str, char *res) {
int count = 0;
if (peek(str,0) == '\\' &&
peek(str,1) == '#' &&
peek(str,2) == 'n' &&
peek(str,3) == 'e' &&
peek(str,4) == 'w' &&
peek(str,5) == 'l' &&
peek(str,6) == 'i' &&
peek(str,7) == 'n' &&
peek(str,8) == 'e') {
*res = '\n';
drop(str,9);
count = 9;
} else if (peek(str,0) == '\\' &&
peek(str,1) == '#' &&
isgraph(peek(str,2))) {
*res = peek(str,2);
drop(str,3);
count = 3;
}
return count;
}
int tok_D(lbm_tokenizer_char_stream_t *str, token_float *result) {
int tok_string(lbm_char_channel_t *ch) {
unsigned int n = 0;
unsigned int m = 0;
char fbuf[128];
bool valid_num = false;
unsigned int len = 0;
char c;
int r = 0;
bool encode = false;
result->type = TOK_TYPE_FLOAT;
r = lbm_channel_peek(ch,0,&c);
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
else if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
if (c != '\"') return TOKENIZER_NO_TOKEN;;
n++;
memset(sym_str, 0 , TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH);
// read string into buffer
r = lbm_channel_peek(ch,n,&c);
while (r == CHANNEL_SUCCESS && c != '\"' &&
len < TOKENIZER_MAX_SYMBOL_AND_STRING_LENGTH) {
if (c == '\\') encode = true;
else {
sym_str[len] = encode ? translate_escape_char(c) : c ;
len++;
encode = false;
}
n ++;
r = lbm_channel_peek(ch, n, &c);
}
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (c != '\"') return TOKENIZER_STRING_ERROR;
n ++;
return (int)n;
}
int tok_char(lbm_char_channel_t *ch, char *res) {
char c;
int r;
r = lbm_channel_peek(ch, 0, &c);
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
if (c != '\\') return TOKENIZER_NO_TOKEN;
r = lbm_channel_peek(ch, 1, &c);
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
if (c != '#') return TOKENIZER_NO_TOKEN;
r = lbm_channel_peek(ch, 2, &c);
if (r == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (r == CHANNEL_END) return TOKENIZER_NO_TOKEN;
*res = c;
return 3;
/* int count = 0; */
/* if (peek(ch,0) == '\\' && */
/* peek(ch,1) == '#' && */
/* peek(ch,2) == 'n' && */
/* peek(ch,3) == 'e' && */
/* peek(ch,4) == 'w' && */
/* peek(ch,5) == 'l' && */
/* peek(ch,6) == 'i' && */
/* peek(ch,7) == 'n' && */
/* peek(ch,8) == 'e') { */
/* *res = '\n'; */
/* drop(ch,9); */
/* count = 9; */
/* } else if (peek(ch,0) == '\\' && */
/* peek(ch,1) == '#' && */
/* isgraph(peek(ch,2))) { */
/* *res = peek(ch,2); */
/* drop(ch,3); */
/* count = 3; */
/* } */
/* return count; */
}
int tok_D(lbm_char_channel_t *ch, token_float *result) {
unsigned int n = 0;
char fbuf[128];
char c;
bool valid_num = false;
int res;
memset(fbuf, 0, 128);
result->type = TOKTYPEF32;
result->negative = false;
if (peek(str, 0) == '-') {
res = lbm_channel_peek(ch, 0, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
else if (res == CHANNEL_END) return 0;
if (c == '-') {
n = 1;
fbuf[0] = 0;
result->negative = true;
}
while ( peek(str,n) >= '0' && peek(str,n) <= '9') n++;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
else if (res == CHANNEL_END) return 0;
while (c >= '0' && c <= '9') {
fbuf[n] = c;
n++;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (res == CHANNEL_END) break;
}
if ( peek(str,n) == '.') n++;
if (c == '.') {
fbuf[n] = c;
n ++;
}
else return 0;
if ( !(peek(str,n) >= '0' && peek(str,n) <= '9')) return 0;
while ( peek(str,n) >= '0' && peek(str,n) <= '9') n++;
res = lbm_channel_peek(ch,n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
else if (res == CHANNEL_END) return 0;
if (!(c >= '0' && c <= '9')) return 0;
unsigned int drop_extra = 0;
if ((peek(str,n) == 'f' &&
peek(str,n+1) == '6' &&
peek(str,n+2) == '4')) {
result->type = TOK_TYPE_DOUBLE;
drop_extra = 3;
while (c >= '0' && c <= '9') {
fbuf[n] = c;
n++;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (res == CHANNEL_END) break;
}
uint32_t tok_res = NOTOKEN;
int type_len = tok_match_fixed_size_tokens(ch, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);
if (type_len == TOKENIZER_NEED_MORE) return type_len;
switch (tok_res) {
case TOKNEEDMORE:
return TOKENIZER_NEED_MORE;
case NOTOKEN:
if ( res == CHANNEL_END || c == ')' || c == ']' || c == '\n' || c == ' ' ) {
result->type = TOKTYPEF32;
break;
} else {
return 0;
}
default:
result->type = tok_res;
break;
}
if ((result->negative && n > 1) ||
(!result->negative && n > 0)) valid_num = true;
if (n > 127) return 0;
else m = n;
if(valid_num) {
unsigned int i;
for (i = 0; i < m; i ++) {
fbuf[i] = get(str);
}
if (n > 127) {
return 0;
}
drop(str,drop_extra);
fbuf[i] = 0;
result->value = (double)strtod(fbuf, NULL);
return (int)n;
if(valid_num) {
result->value = (double)strtod(fbuf,NULL);
return (int)n + type_len;
}
return 0;
}
void clean_whitespace(lbm_tokenizer_char_stream_t *str) {
bool clean_whitespace(lbm_char_channel_t *ch) {
bool clean_whitespace = true;
while ( clean_whitespace ){
if ( peek(str,0) == ';' ) {
while ( more(str) && peek(str, 0) != '\n') {
drop(str,1);
bool cleaning_whitespace = true;
char c;
int r;
while (cleaning_whitespace) {
if (lbm_channel_comment(ch)) {
while (true) {
r = lbm_channel_peek(ch, 0, &c);
if (r == CHANNEL_END) {
lbm_channel_set_comment(ch, false);
cleaning_whitespace = false;
break;
}
if (r == CHANNEL_MORE) {
return false;
}
lbm_channel_drop(ch,1);
if (c == '\n') {
lbm_channel_set_comment(ch, false);
break;
}
}
} else if ( isspace(peek(str,0))) {
drop(str,1);
} else {
clean_whitespace = false;
}
do {
r = lbm_channel_peek(ch, 0, &c);
if (r == CHANNEL_MORE) {
return false;
} else if (r == CHANNEL_END) {
return true;
}
if (c == ';') {
lbm_channel_set_comment(ch, true);
break;
}
if (isspace(c)) {
lbm_channel_drop(ch,1);
} else {
cleaning_whitespace = false;
}
} while (cleaning_whitespace);
}
return true;
}
int tok_integer(lbm_tokenizer_char_stream_t *str, token_int *result ) {
int tok_integer(lbm_char_channel_t *ch, token_int *result ) {
uint64_t acc = 0;
unsigned int n = 0;
bool valid_num = false;
result->negative = false;
if (peek(str, 0) == '-') {
char c;
int res;
result-> negative = false;
res = lbm_channel_peek(ch, 0, &c);
if (res == CHANNEL_MORE) {
return TOKENIZER_NEED_MORE;
} else if (res == CHANNEL_END) {
return 0;
}
if (c == '-') {
n = 1;
result->negative = true;
}
// Check if hex notation is used
if (peek(str,n) == '0' &&
(peek(str,n+1) == 'x' || peek(str,n+1) == 'X')) {
n+= 2;
while ( (peek(str,n) >= '0' && peek(str,n) <= '9') ||
(peek(str,n) >= 'a' && peek(str,n) <= 'f') ||
(peek(str,n) >= 'A' && peek(str,n) <= 'F')){
bool hex = false;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_SUCCESS && c == '0') {
res = lbm_channel_peek(ch, n + 1, &c);
if ( res == CHANNEL_SUCCESS && (c == 'x' || c == 'X')) {
hex = true;
} else if (res == CHANNEL_MORE) {
return TOKENIZER_NEED_MORE;
}
} else if (res == CHANNEL_MORE) {
return TOKENIZER_NEED_MORE;
}
if (hex) {
n += 2;
res = lbm_channel_peek(ch,n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
else if (res == CHANNEL_END) return 0;
while ((c >= '0' && c <= '9') ||
(c >= 'a' && c <= 'f') ||
(c >= 'A' && c <= 'F')) {
uint32_t val; /* values between 0 and 16 */
if (peek(str,n) >= 'a' && peek(str,n) <= 'f') {
val = 10 + (uint32_t)(peek(str,n) - 'a');
} else if (peek(str,n) >= 'A' && peek(str,n) <= 'F') {
val = 10 + (uint32_t)(peek(str,n) - 'A');
if (c >= 'a' && c <= 'f') {
val = 10 + (uint32_t)c - 'a';
} else if (c >= 'A' && c <= 'F') {
val = 10 + (uint32_t)(c - 'A');
} else {
val = (uint32_t)peek(str,n) - '0';
val = (uint32_t)c - '0';
}
acc = (acc * 0x10) + val;
n++;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (res == CHANNEL_END) break;
}
} else {
while ( peek(str,n) >= '0' && peek(str,n) <= '9' ){
acc = (acc*10) + (uint32_t)(peek(str,n) - '0');
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
while (c >= '0' && c <= '9') {
acc = (acc*10) + (uint32_t)(c - '0');
n++;
res = lbm_channel_peek(ch, n, &c);
if (res == CHANNEL_MORE) return TOKENIZER_NEED_MORE;
if (res == CHANNEL_END) break;
}
}
unsigned int drop_type_str = 0;
if (peek(str,n) == 'u' &&
peek(str,n+1) == '6' &&
peek(str,n+2) == '4') {
drop_type_str = 3;
result->type = TOK_TYPE_U64;
} else if (peek(str,n) == 'i' &&
peek(str,n+1) == '6' &&
peek(str,n+2) == '4') {
drop_type_str = 3;
result->type = TOK_TYPE_I64;
} else if (peek(str,n) == 'u' &&
peek(str,n+1) == '3' &&
peek(str,n+2) == '2') {
drop_type_str = 3;
result->type = TOK_TYPE_U32;
} else if (peek(str,n) == 'i' &&
peek(str,n+1) == '3' &&
peek(str,n+2) == '2') {
drop_type_str = 3;
result->type = TOK_TYPE_I32;
} else if (peek(str,n) == 'i') {
drop_type_str = 1;
result->type = TOK_TYPE_I;
} else if (peek(str,n) == 'u') {
drop_type_str = 1;
result->type = TOK_TYPE_U;
} else if (peek(str,n) == 'b') {
drop_type_str = 1;
result->type = TOK_TYPE_BYTE;
} else {
result->type = TOK_TYPE_I;
if (n == 0) return 0;
result->type = TOKTYPEI;
uint32_t tok_res = NOTOKEN;
int type_len = tok_match_fixed_size_tokens(ch, type_qual_table, n, NUM_TYPE_QUALIFIERS, &tok_res);
if (type_len == TOKENIZER_NEED_MORE) return type_len;
switch (tok_res) {
case TOKNEEDMORE:
return TOKENIZER_NEED_MORE;
case NOTOKEN:
if ( res == CHANNEL_END || c == ')' || c == ']' || c == '\n' || c == ' ' ) {
result->type = TOKTYPEI;
break;
} else {
return 0;
}
default:
result->type = tok_res;
break;
}
if ((result->negative && n > 1) ||
(!result->negative && n > 0)) valid_num = true;
if (valid_num) {
drop(str,n + drop_type_str);
//lbm_channel_drop(ch,n + drop_type_str);
result->value = acc;
return (int)n; /*check that isnt so high that it becomes a negative number when casted */
return (int)n + type_len;
}
return 0;
}
lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str) {
lbm_value lbm_get_next_token(lbm_char_channel_t *ch, bool peek) {
char c_val;
int n = 0;
if (!more(str)) {
if (!lbm_channel_more(ch) && lbm_channel_is_empty(ch)) {
return lbm_enc_sym(SYM_TOKENIZER_DONE);
}
// Eat whitespace and comments.
clean_whitespace(str);
if (!clean_whitespace(ch)) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
// Check for end of string again
if (!more(str)) {
if (!lbm_channel_more(ch) && lbm_channel_is_empty(ch)) {
return lbm_enc_sym(SYM_TOKENIZER_DONE);
}
lbm_value res = lbm_enc_sym(SYM_RERROR);
uint32_t match;
match = tok_match_fixed_size_tokens(str);
if (match > 0) {
n = tok_match_fixed_size_tokens(ch,
fixed_size_tokens,
0,
NUM_FIXED_SIZE_TOKENS,
&match);
if (n > 0) {
if (!peek) {
if (!lbm_channel_drop(ch, (unsigned int)n)) {
// Really should not happen (bug in channel implementation)
}
}
switch (match) {
case TOKOPENPAR:
res = lbm_enc_sym(SYM_OPENPAR);
@ -542,10 +669,13 @@ lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str) {
break;
}
return res;
} else if (n < 0) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
n = tok_string(str);
n = tok_string(ch);
if (n >= 2) {
if (!peek) lbm_channel_drop(ch, (unsigned int)n);
// TODO: Proper error checking here!
// TODO: Check if anything has to be allocated for the empty string
lbm_heap_allocate_array(&res, (unsigned int)(n-2)+1, LBM_TYPE_CHAR);
@ -554,57 +684,68 @@ lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str) {
memset(data, 0, (unsigned int)((n-2)+1) * sizeof(char));
memcpy(data, sym_str, (unsigned int)(n - 2) * sizeof(char));
return res;
} else if (n < 0) {
// The string is too long error
return res;
} else if (n == TOKENIZER_NEED_MORE) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
} else if (n == TOKENIZER_STRING_ERROR) {
return lbm_enc_sym(SYM_RERROR);
}
token_float f_val;
if (tok_D(str, &f_val)) {
n = tok_D(ch, &f_val);
if (n > 0) {
if (!peek) lbm_channel_drop(ch, (unsigned int)n);
switch (f_val.type) {
case TOK_TYPE_FLOAT:
case TOKTYPEF32:
return lbm_enc_float((float)f_val.value);
case TOK_TYPE_DOUBLE:
case TOKTYPEF64:
return lbm_enc_double(f_val.value);
}
} else if ( n < 0) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
token_int int_result;
if (tok_integer(str, &int_result)) {
n = tok_integer(ch, &int_result);
if (n > 0) {
if (!peek) lbm_channel_drop(ch, (unsigned int)n);
switch (int_result.type) {
case TOK_TYPE_BYTE:
case TOKTYPEBYTE:
return lbm_enc_char((char)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_I:
case TOKTYPEI:
return lbm_enc_i((lbm_int)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_U:
case TOKTYPEU:
return lbm_enc_u((lbm_uint)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_I32:
case TOKTYPEI32:
return lbm_enc_i32((lbm_int)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_U32:
case TOKTYPEU32:
return lbm_enc_u32((lbm_uint)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_I64:
case TOKTYPEI64:
return lbm_enc_i64((int64_t)(int_result.negative ? -int_result.value : int_result.value));
break;
case TOK_TYPE_U64:
case TOKTYPEU64:
return lbm_enc_u64((uint64_t)(int_result.negative ? -int_result.value : int_result.value));
break;
default:
return lbm_enc_sym(SYM_RERROR);
break;
}
} else if (n < 0 ) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
n = tok_symbol(str);
n = tok_symbol(ch);
if (n > 0) {
if (!peek) lbm_channel_drop(ch,(unsigned int)n);
lbm_uint symbol_id;
if (lbm_get_symbol_by_name(sym_str, &symbol_id)) {
@ -627,91 +768,29 @@ lbm_value lbm_get_next_token(lbm_tokenizer_char_stream_t *str) {
}
return res;
} else if (n < 0) {
// Symbol string is too long error
return res;
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
if (tok_char(str, &c_val)) {
n = tok_char(ch, &c_val);
if (n > 0) {
if (!peek) lbm_channel_drop(ch,(unsigned int)n);
return lbm_enc_char(c_val);
} else if (n < 0) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
// Status of "more" can have changed between
// the start of this function and this location.
if (lbm_channel_more(ch)) {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
} else {
if (lbm_channel_is_empty(ch)) {
return lbm_enc_sym(SYM_TOKENIZER_DONE);
} else {
return lbm_enc_sym(SYM_TOKENIZER_WAIT);
}
}
return res;
}
bool more_string(lbm_tokenizer_char_stream_t *str) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
if ( s->pos > strlen(s->str)) {
return false;
} else {
return s->str[s->pos] != 0;
}
}
char get_string(lbm_tokenizer_char_stream_t *str) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
char c = s->str[s->pos];
s->pos ++;
s->column ++;
return c;
}
bool put_string(lbm_tokenizer_char_stream_t *str, char c) {
(void) str;
(void) c;
return false;
}
char peek_string(lbm_tokenizer_char_stream_t *str, unsigned int n) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
// TODO error checking ?? how ?
char c = s->str[s->pos + n];
return c;
}
void drop_string(lbm_tokenizer_char_stream_t *str, unsigned int n) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
for (unsigned int i = 0; i < n; i ++) {
if (s->str[s->pos + i] == '\n') {
s->row++;
s->column = 1;
} else {
s->column ++;
}
}
s->pos += n;
}
unsigned int row_string(lbm_tokenizer_char_stream_t *str) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
return s->row;
}
unsigned int column_string(lbm_tokenizer_char_stream_t *str) {
lbm_tokenizer_string_state_t *s =
(lbm_tokenizer_string_state_t *)str->state;
return s->column;
}
void lbm_create_char_stream_from_string(lbm_tokenizer_string_state_t *state,
lbm_tokenizer_char_stream_t *char_stream,
const char *string){
state->str = string;
state->pos = 0;
state->row = 1;
state->column = 1;
char_stream->state = state;
char_stream->more = more_string;
char_stream->peek = peek_string;
char_stream->drop = drop_string;
char_stream->get = get_string;
char_stream->put = put_string;
char_stream->row = row_string;
char_stream->column = column_string;
}

6
tests/run_tests.sh
View File

@ -8,9 +8,9 @@ make
echo "PERFORMING TESTS:"
expected_fails=("test_lisp_code_cps -h 1024 test_take_iota_0.lisp"
"test_lisp_code_cps -c -h 1024 test_take_iota_0.lisp"
"test_lisp_code_cps -s -h 1024 test_take_iota_0.lisp"
"test_lisp_code_cps -h 512 test_take_iota_0.lisp"
"test_lisp_code_cps -c -h 512 test_take_iota_0.lisp"
"test_lisp_code_cps -s -h 512 test_take_iota_0.lisp"
)
@ -44,7 +44,7 @@ done
#"test_lisp_code_cps_nc"
for prg in "test_lisp_code_cps" ; do
for arg in "-h 32768" "-c -h 32768" "-h 16384" "-c -h 16384" "-h 8192" "-c -h 8192" "-h 4096" "-c -h 4096" "-h 2048" "-c -h 2048" "-h 1024" "-c -h 1024" "-h 512" "-c -h 512" ; do
for arg in "-h 32768" "-s -h 32768" "-h 16384" "-s -h 16384" "-h 8192" "-s -h 8192" "-h 4096" "-s -h 4096" "-h 2048" "-s -h 2048" "-h 1024" "-s -h 1024" "-h 512" "-s -h 512" ; do
for lisp in *.lisp; do
./$prg $arg $lisp

4
tests/test_array_syntax_14.lisp Normal file
View File

@ -0,0 +1,4 @@
(def arr [type-i32 1 2 3 4 5 6])
(= (array-size arr) 6)

4
tests/test_array_syntax_15.lisp Normal file
View File

@ -0,0 +1,4 @@
(define arr [type-i32 1 2 3 4 5 6 7])
(= (array-size arr) 7)

4
tests/test_array_syntax_16.lisp Normal file
View File

@ -0,0 +1,4 @@
(define arr [ 1 2 3 4 5 6 ])
(= (array-size arr) 6)

4
tests/test_array_syntax_17.lisp Normal file
View File

@ -0,0 +1,4 @@
(define arr [1 2 3 4 5 6 7])
(= (array-size arr) 7)

100
tests/test_lisp_code_cps.c
View File

@ -27,6 +27,7 @@
#include "lispbm.h"
#include "extensions/array_extensions.h"
#include "lbm_channel.h"
#define WAIT_TIMEOUT 2500
@ -42,13 +43,17 @@ extension_fptr extension_storage[EXTENSION_STORAGE_SIZE];
lbm_value variable_storage[VARIABLE_STORAGE_SIZE];
/* Tokenizer state for strings */
static lbm_tokenizer_string_state_t string_tok_state;
//static lbm_tokenizer_string_state_t string_tok_state;
/* Tokenizer statefor compressed data */
static tokenizer_compressed_state_t comp_tok_state;
//static tokenizer_compressed_state_t comp_tok_state;
/* shared tokenizer */
static lbm_tokenizer_char_stream_t string_tok;
//static lbm_tokenizer_char_stream_t string_tok;
static lbm_char_channel_t string_tok;
static lbm_string_channel_state_t string_tok_state;
static lbm_buffered_channel_state_t buffered_tok_state;
void *eval_thd_wrapper(void *v) {
(void)v;
@ -201,7 +206,7 @@ LBM_EXTENSION(ext_numbers, args, argn) {
bool b = true;
for (int i = 0; i < argn; i ++) {
for (unsigned int i = 0; i < argn; i ++) {
if (!lbm_is_number(args[i])) {
b = false;
break;
@ -217,22 +222,26 @@ int main(int argc, char **argv) {
int res = 0;
unsigned int heap_size = 8 * 1024 * 1024; // 8 Megabytes is standard
bool compress_decompress = false;
// bool compress_decompress = false;
bool stream_source = false;
pthread_t lispbm_thd;
lbm_cons_t *heap_storage = NULL;
int c;
opterr = 1;
while (( c = getopt(argc, argv, "gch:")) != -1) {
while (( c = getopt(argc, argv, "gsch:")) != -1) {
switch (c) {
case 'h':
heap_size = (unsigned int)atoi((char *)optarg);
break;
case 'c':
compress_decompress = true;
break;
// case 'c':
//compress_decompress = true;
//break;
case 's':
stream_source = true;
case '?':
break;
default:
@ -241,7 +250,8 @@ int main(int argc, char **argv) {
}
printf("------------------------------------------------------------\n");
printf("Heap size: %u\n", heap_size);
printf("Compression: %s\n", compress_decompress ? "yes" : "no");
printf("Streaming source: %s\n", stream_source ? "yes" : "no");
// printf("Compression: %s\n", compress_decompress ? "yes" : "no");
printf("------------------------------------------------------------\n");
if (argc - optind < 1) {
@ -395,30 +405,37 @@ int main(int argc, char **argv) {
/* sleep_callback(1000); */
/* } */
char *compressed_code;
if (compress_decompress) {
uint32_t compressed_size = 0;
compressed_code = lbm_compress(code_buffer, &compressed_size);
if (!compressed_code) {
printf("Error compressing code\n");
return 0;
}
//char decompress_code[8192];
char decompress_code[64000];
/* char *compressed_code; */
/* if (compress_decompress) { */
/* uint32_t compressed_size = 0; */
/* compressed_code = lbm_compress(code_buffer, &compressed_size); */
/* if (!compressed_code) { */
/* printf("Error compressing code\n"); */
/* return 0; */
/* } */
/* //char decompress_code[8192]; */
/* char decompress_code[64000]; */
lbm_decompress(decompress_code, 64000, compressed_code);
printf("\n\nDECOMPRESS TEST: %s\n\n", decompress_code);
/* lbm_decompress(decompress_code, 64000, compressed_code); */
/* printf("\n\nDECOMPRESS TEST: %s\n\n", decompress_code); */
lbm_create_char_stream_from_compressed(&comp_tok_state,
&string_tok,
compressed_code);
/* lbm_create_char_stream_from_compressed(&comp_tok_state, */
/* &string_tok, */
/* compressed_code); */
/* } else { */
//lbm_create_char_stream_from_string(&string_tok_state,
// &string_tok,
// code_buffer);
if (stream_source) {
lbm_create_buffered_char_channel(&buffered_tok_state,
&string_tok);
} else {
lbm_create_char_stream_from_string(&string_tok_state,
&string_tok,
code_buffer);
lbm_create_string_char_channel(&string_tok_state,
&string_tok,
code_buffer);
}
//}
lbm_set_ctx_done_callback(context_done_callback);
cid = lbm_load_and_eval_program(&string_tok);
@ -430,18 +447,35 @@ int main(int argc, char **argv) {
lbm_continue_eval();
if (stream_source) {
int i = 0;
while (true) {
if (code_buffer[i] == 0) {
lbm_channel_writer_close(&string_tok);
break;
}
int ch_res = lbm_channel_write(&string_tok, code_buffer[i]);
if (ch_res == CHANNEL_SUCCESS) {
//printf("wrote: %c\n", code_buffer[i]);
i ++;
} if (ch_res == CHANNEL_READER_CLOSED) {
break;
}
sleep_callback(2);
}
}
while (!experiment_done) {
sleep_callback(1000);
}
lbm_pause_eval();
while(lbm_get_eval_state() != EVAL_CPS_STATE_PAUSED);
if (compress_decompress) {
free(compressed_code);
}
/* if (compress_decompress) { */
/* free(compressed_code); */
/* } */
free(heap_storage);

35
tests/test_n_times.sh Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
echo "BUILDING"
make clean
make
n=$1
test=$2
result=0
success_count=0
fail_count=0
for ((i=0; i<n; i++));
do
./test_lisp_code_cps -s -h 32000 $test
result=$?
echo "------------------------------------------------------------"
echo Test: $i
if [ $result -eq 1 ]
then
success_count=$((success_count+1))
echo SUCCESS
else
fail_count=$((fail_count+1))
echo FAILED
break
fi
echo "------------------------------------------------------------"
done
echo OK: $success_count
echo FAILED: $fail_count

2
tests/test_symbol_0.lisp Normal file
View File

@ -0,0 +1,2 @@
(eq 'apa 'apa)

4
tests/test_symbol_1.lisp Normal file
View File

@ -0,0 +1,4 @@
(eq (type-of 'abcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefghabcdefgh) type-symbol)