Skip to content

Commit

Permalink
Pull out pretokenisation on SAVE_ON_FLASH boards (pretokenised code c…
Browse files Browse the repository at this point in the history
…an be executed, it's just not tokenised in Espruino - saves 1kb)
gfwilliams committed Jan 26, 2024
1 parent aec666e commit f80bc1b
Showing 7 changed files with 22 additions and 8 deletions.
1 change: 1 addition & 0 deletions ChangeLog
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
Fix issue requesting constructor of builtin before the class itself has been accessed (fix #2451)
Allow pretokenisation to store raw, unencoded Strings for speed/efficiency
Automatic decoding and pretokenisation of 'atob(".....")' strings
Pull out pretokenisation on SAVE_ON_FLASH boards (pretokenised code can be executed, it's just not tokenised in Espruino - saves 1kb)

2v20 : Ensure String.charCodeAt returns NaN for out of bounds chars
Bangle.js2: When rendering overlays, *do not* use the current FG/BG color for 1 bit overlays
1 change: 1 addition & 0 deletions README_BuildProcess.md
Original file line number Diff line number Diff line change
@@ -157,6 +157,7 @@ These are set automatically when `SAVE_ON_FLASH` is set (see `jsutils.h`)
* `ESPR_NO_LINE_NUMBERS` - disable storing and reporting of Line Numbers. Usually these take 1 var per function, but if we're executing a function from flash we can just work it out from the file when needed
* `ESPR_NO_LET_SCOPING` - don't create scopes for `let` (treat it like `var`, which was the 2v13 and earlier behaviour)
* `ESPR_NO_PROMISES` - Don't include promise-handling functions
* `ESPR_NO_PRETOKENISE` - Don't include code to pretokenise functions marked with `"ram"` - code pretokenised in the IDE can still be executed


### chip
10 changes: 6 additions & 4 deletions src/jsflags.h
Original file line number Diff line number Diff line change
@@ -19,16 +19,18 @@
typedef enum {
JSF_NONE,
JSF_DEEP_SLEEP = 1<<0, ///< Allow deep sleep modes (also set by setDeepSleep)
JSF_PRETOKENISE = 1<<1, ///< When adding functions, pre-minify them and tokenise reserved words
JSF_UNSAFE_FLASH = 1<<2, ///< Some platforms stop writes/erases to interpreter memory to stop you bricking the device accidentally - this removes that protection
JSF_UNSYNC_FILES = 1<<3, ///< When accessing files, *don't* flush all data to the SD card after each command. Faster, but risky if power is lost
JSF_UNSAFE_FLASH = 1<<1, ///< Some platforms stop writes/erases to interpreter memory to stop you bricking the device accidentally - this removes that protection
JSF_UNSYNC_FILES = 1<<2, ///< When accessing files, *don't* flush all data to the SD card after each command. Faster, but risky if power is lost
#ifndef ESPR_NO_PRETOKENISE
JSF_PRETOKENISE = 1<<3, ///< When adding functions, pre-minify them and tokenise reserved words (adding "ram" at the start does this too)
#endif
#ifdef ESPR_JIT
JSF_JIT_DEBUG = 1<<4, ///< When JIT enabled,
#endif
} PACKED_FLAGS JsFlags;


#define JSFLAG_NAMES "deepSleep\0pretokenise\0unsafeFlash\0unsyncFiles\0jitDebug\0"
#define JSFLAG_NAMES "deepSleep\0unsafeFlash\0unsyncFiles\0pretokenise\0jitDebug\0"
// NOTE: \0 also added by compiler - two \0's are required!

extern volatile JsFlags jsFlags;
3 changes: 3 additions & 0 deletions src/jslex.c
Original file line number Diff line number Diff line change
@@ -1157,6 +1157,7 @@ bool jslMatch(int expected_tk) {
return true;
}

#ifndef ESPR_NO_PRETOKENISE
// When minifying/pretokenising, do we need to insert a space between these tokens?
static bool jslPreserveSpaceBetweenTokens(int lastTk, int newTk) {
// spaces between numbers/IDs
@@ -1290,6 +1291,8 @@ JsVar *jslNewTokenisedStringFromLexer(JslCharPos *charFrom, size_t charTo) {
return var;
}

#endif // ESPR_NO_PRETOKENISE

JsVar *jslNewStringFromLexer(JslCharPos *charFrom, size_t charTo) {
// Original method - just copy it verbatim
size_t maxLength = charTo + 1 - jsvStringIteratorGetIndex(&charFrom->it);
2 changes: 2 additions & 0 deletions src/jslex.h
Original file line number Diff line number Diff line change
@@ -204,8 +204,10 @@ void jslGetNextToken(); ///< Get the text token from our text string
/// Create a new STRING from part of the lexer
JsVar *jslNewStringFromLexer(JslCharPos *charFrom, size_t charTo);

#ifndef ESPR_NO_PRETOKENISE
/// Create a new STRING from part of the lexer - keywords get tokenised
JsVar *jslNewTokenisedStringFromLexer(JslCharPos *charFrom, size_t charTo);
#endif

/// Return the line number at the current character position (this isn't fast as it searches the string)
unsigned int jslGetLineNumber();
12 changes: 8 additions & 4 deletions src/jsparse.c
Original file line number Diff line number Diff line change
@@ -344,11 +344,14 @@ NO_INLINE bool jspeFunctionDefinitionInternal(JsVar *funcVar, bool expressionOnl
JsVar *tokenValue = jslGetTokenValueAsVar();
if (jsvIsStringEqual(tokenValue, "compiled")) {
jsWarn("Function marked with \"compiled\" uploaded in source form");
} else if (jsvIsStringEqual(tokenValue, "ram")) {
}
#ifndef ESPR_NO_PRETOKENISE
else if (jsvIsStringEqual(tokenValue, "ram")) {
JSP_ASSERT_MATCH(LEX_STR);
if (lex->tk==';') JSP_ASSERT_MATCH(';');
forcePretokenise = true;
}
#endif
#ifdef ESPR_JIT
else if (jsvIsStringEqual(tokenValue, "jit")) {
JslCharPos funcCodeStart;
@@ -443,11 +446,12 @@ NO_INLINE bool jspeFunctionDefinitionInternal(JsVar *funcVar, bool expressionOnl
funcCodeVar = jsvNewFlashString(lex->sourceVar->varData.nativeStr.ptr + s, (unsigned int)(lastTokenEnd - s));
#endif
} else {
if (jsfGetFlag(JSF_PRETOKENISE) || forcePretokenise) {
#ifndef ESPR_NO_PRETOKENISE
if (jsfGetFlag(JSF_PRETOKENISE) || forcePretokenise)
funcCodeVar = jslNewTokenisedStringFromLexer(&funcBegin, (size_t)lastTokenEnd);
} else {
else
#endif
funcCodeVar = jslNewStringFromLexer(&funcBegin, (size_t)lastTokenEnd);
}
}
jsvAddNamedChildAndUnLock(funcVar, funcCodeVar, JSPARSE_FUNCTION_CODE_NAME);
// scope var
1 change: 1 addition & 0 deletions src/jsutils.h
Original file line number Diff line number Diff line change
@@ -51,6 +51,7 @@
#define ESPR_NO_CLASSES 1
#define ESPR_NO_ARROW_FN 1
#define ESPR_NO_REGEX 1
#define ESPR_NO_PRETOKENISE 1
#define ESPR_NO_TEMPLATE_LITERAL 1
#define ESPR_NO_SOFTWARE_SERIAL 1
#ifndef ESPR_NO_SOFTWARE_I2C

0 comments on commit f80bc1b

Please sign in to comment.