diff --git a/ChangeLog b/ChangeLog index eb35bb1423..a4eb31ad2d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,8 @@ Graphics: Improve PBF font loading to handle v3, plus Espruino extension to handle >10k glyphs in one file Graphics: Graphics.stringMetrics now returns 'unknownChars' to indicate if a font can't render a character in the String Fix unicode in object accesses, eg c["\u00FC"]=42 (fix #2429) + Storage: Storage.writeJSON now escapes with a more compact `\x##`/`\#` character escape notation (still valid JSON) + This allows Espruino to save non-unicode strings with characters in the unicode range, and to also re-load them as non-unicode 2v19 : Fix Object.values/entries for numeric keys after 2v18 regression (fix #2375) nRF52: for SD>5 use static buffers for advertising and scan response data (#2367) diff --git a/src/jsutils.c b/src/jsutils.c index 4ba36c4dbb..634ac6c5d8 100644 --- a/src/jsutils.c +++ b/src/jsutils.c @@ -752,8 +752,8 @@ JsVarFloat wrapAround(JsVarFloat val, JsVarFloat size) { * * `%s` = string (char *) * * `%c` = char * * `%v` = JsVar * (doesn't have to be a string - it'll be converted) - * * `%q` = JsVar * (in quotes, and escaped) - * * `%Q` = JsVar * (in quotes, and escaped the JSON subset of escape chars) + * * `%q` = JsVar * (in quotes, and escaped with \uXXXX,\xXX,\X whichever makes sense) + * * `%Q` = JsVar * (in quotes, and escaped with only \uXXXX) * * `%j` = Variable printed as JSON * * `%t` = Type of variable * * `%p` = Pin @@ -824,6 +824,7 @@ void vcbprintf( bool isJSONStyle = fmtChar=='Q'; if (quoted) user_callback("\"",user_data); JsVar *v = jsvAsString(va_arg(argp, JsVar*)); + if (jsvIsUTF8String(v)) isJSONStyle=true; // if it's a UTF8 string make sure we escape in UTF8 form to force Espruino to re-create it as a UTF8 string when parsing buf[1] = 0; if (jsvIsString(v)) { JsvStringIterator it; diff --git a/src/jsutils.h b/src/jsutils.h index 682905d942..c7a804724b 100755 --- a/src/jsutils.h +++ b/src/jsutils.h @@ -581,8 +581,8 @@ typedef void (*vcbprintf_callback)(const char *str, void *user_data); * * `%s` = string (char *) * * `%c` = char * * `%v` = JsVar * (doesn't have to be a string - it'll be converted) - * * `%q` = JsVar * (in quotes, and escaped) - * * `%Q` = JsVar * (in quotes, and escaped the JSON subset of escape chars) + * * `%q` = JsVar * (in quotes, and escaped with \uXXXX,\xXX,\X whichever makes sense) + * * `%Q` = JsVar * (in quotes, and escaped with only \uXXXX) * * `%j` = Variable printed as JSON * * `%t` = Type of variable * * `%p` = Pin diff --git a/src/jswrap_json.c b/src/jswrap_json.c index 731a61df9e..2b133481d6 100644 --- a/src/jswrap_json.c +++ b/src/jswrap_json.c @@ -293,7 +293,7 @@ static bool jsfGetJSONForObjectItWithCallback(JsvObjectIterator *it, JSONFlags f if (isIDString(buf)) addQuotes=false; } } - cbprintf(user_callback, user_data, addQuotes?((flags&JSON_JSON_COMPATIBILE)?"%Q%s":"%q%s"):"%v%s", index, (flags&JSON_PRETTY)?": ":":"); + cbprintf(user_callback, user_data, addQuotes?((flags&JSON_ALL_UNICODE_ESCAPE)?"%Q%s":"%q%s"):"%v%s", index, (flags&JSON_PRETTY)?": ":":"); if (first) first = false; jsfGetJSONWithCallback(item, index, nflags, whitespace, user_callback, user_data); @@ -482,7 +482,7 @@ void jsfGetJSONWithCallback(JsVar *var, JsVar *varName, JSONFlags flags, const c cbprintf(user_callback, user_data, "function "); jsfGetJSONForFunctionWithCallback(var, nflags, user_callback, user_data); } - } else if ((jsvIsString(var) && !jsvIsName(var)) || ((flags&JSON_JSON_COMPATIBILE)&&jsvIsPin(var))) { + } else if ((jsvIsString(var) && !jsvIsName(var)) || ((flags&JSON_PIN_TO_STRING)&&jsvIsPin(var))) { if ((flags&JSON_LIMIT) && jsvGetStringLength(var)>JSON_LIMIT_STRING_AMOUNT) { // if the string is too big, split it and put dots in the middle JsVar *var1 = jsvNewFromStringVar(var, 0, JSON_LIMITED_STRING_AMOUNT); @@ -490,9 +490,9 @@ void jsfGetJSONWithCallback(JsVar *var, JsVar *varName, JSONFlags flags, const c cbprintf(user_callback, user_data, "%q%s%q", var1, JSON_LIMIT_TEXT, var2); jsvUnLock2(var1, var2); } else { - cbprintf(user_callback, user_data, (flags&JSON_JSON_COMPATIBILE)?"%Q":"%q", var); + cbprintf(user_callback, user_data, (flags&JSON_ALL_UNICODE_ESCAPE)?"%Q":"%q", var); } - } else if ((flags&JSON_JSON_COMPATIBILE) && jsvIsFloat(var) && !isfinite(jsvGetFloat(var))) { + } else if ((flags&JSON_NO_NAN) && jsvIsFloat(var) && !isfinite(jsvGetFloat(var))) { cbprintf(user_callback, user_data, "null"); } else { cbprintf(user_callback, user_data, "%v", var); diff --git a/src/jswrap_json.h b/src/jswrap_json.h index 0966c32db5..b4f5ab245c 100644 --- a/src/jswrap_json.h +++ b/src/jswrap_json.h @@ -32,14 +32,13 @@ typedef enum { JSON_ARRAYBUFFER_AS_ARRAY = 128, //< dump arraybuffers as arrays JSON_SHOW_OBJECT_NAMES = 256, //< Show 'Promise {}'/etc for objects if the type is global JSON_DROP_QUOTES = 512, //< When outputting objects, drop quotes for alphanumeric field names - JSON_JSON_COMPATIBILE = 1024, /**< - Only use unicode for escape characters - needed for JSON compatibility - Don't output NaN for NaN numbers, only 'null' - Convert pins to Strings - */ - JSON_ALLOW_TOJSON = 2048, //< If there's a .toJSON function in an object, use it and parse that + JSON_PIN_TO_STRING = 1024, //< Convert pins to Strings + JSON_ALL_UNICODE_ESCAPE = 2048, //< Only use unicode \xXXXX for escape characters, not \xXX or \X + JSON_NO_NAN = 4096, //< Don't output NaN for NaN numbers, only 'null' + JSON_JSON_COMPATIBILE = JSON_PIN_TO_STRING|JSON_ALL_UNICODE_ESCAPE|JSON_NO_NAN, //< specific stuff needed for compatibility + JSON_ALLOW_TOJSON = 8192, //< If there's a .toJSON function in an object, use it and parse that // ... - JSON_INDENT = 4096, // MUST BE THE LAST ENTRY IN JSONFlags - we use this to count the amount of indents + JSON_INDENT = 16384, // MUST BE THE LAST ENTRY IN JSONFlags - we use this to count the amount of indents } JSONFlags; /* This is like jsfGetJSONWithCallback, but handles ONLY functions (and does not print the initial 'function' text) */ diff --git a/src/jswrap_storage.c b/src/jswrap_storage.c index 5963598649..6af1ac8508 100644 --- a/src/jswrap_storage.c +++ b/src/jswrap_storage.c @@ -285,13 +285,23 @@ disappear when the device resets or power is lost. Simply write `require("Storage").writeJSON("MyFile", [1,2,3])` to write a new file, and `require("Storage").readJSON("MyFile")` to read it. -This is equivalent to: `require("Storage").write(name, JSON.stringify(data))` +This is (almost) equivalent to: `require("Storage").write(name, JSON.stringify(data))` **Note:** This function should be used with normal files, and not `StorageFile`s created with `require("Storage").open(filename, ...)` + +**Note:** Normally `JSON.stringify` converts any non-standard character to an escape code with `\uXXXX`, but +as of Espruino 2v20, when writing to a file we use the most compact form, like `\xXX` or `\X`. This saves +space and is faster, but also means that if a String wasn't a UTF8 string but contained characters in the UTF8 codepoint range, +when saved it won't end up getting reloaded as a UTF8 string. */ bool jswrap_storage_writeJSON(JsVar *name, JsVar *data) { - JsVar *d = jswrap_json_stringify(data,0,0); + JsVar *d = jsvNewFromEmptyString(); + if (!d) return false; + /* Don't call jswrap_json_stringify directly because we want to ensure we don't use JSON_JSON_COMPATIBILE, so + String escapes like `\xFC` stay as `\xFC` and not `\u00FC` to save space and help with unicode compatibility + */ + jsfGetJSON(data, d, (JSON_IGNORE_FUNCTIONS|JSON_NO_UNDEFINED|JSON_ARRAYBUFFER_AS_ARRAY|JSON_JSON_COMPATIBILE) &~JSON_ALL_UNICODE_ESCAPE); bool r = jsfWriteFile(jsfNameFromVar(name), d, JSFF_NONE, 0, 0); jsvUnLock(d); return r; @@ -426,7 +436,7 @@ void jswrap_storage_debug() { "name" : "getFree", "params" : [ ["checkInternalFlash","bool","Check the internal flash (rather than external SPI flash). Default false, so will check external storage"] - ], + ], "generate" : "jswrap_storage_getFree", "return" : ["int","The amount of free bytes"] } @@ -451,7 +461,7 @@ int jswrap_storage_getFree(bool checkInternalFlash) { "name" : "getStats", "params" : [ ["checkInternalFlash","bool","Check the internal flash (rather than external SPI flash). Default false, so will check external storage"] - ], + ], "generate" : "jswrap_storage_getStats", "return" : ["JsVar","An object containing info about the current Storage system"] } @@ -476,8 +486,8 @@ JsVar *jswrap_storage_getStats(bool checkInternalFlash) { uint32_t addr = 0; #ifdef FLASH_SAVED_CODE2_START addr = checkInternalFlash ? FLASH_SAVED_CODE_START : FLASH_SAVED_CODE2_START; - -#endif + +#endif JsfStorageStats stats = jsfGetStorageStats(addr, true); jsvObjectSetChildAndUnLock(o, "totalBytes", jsvNewFromInteger((JsVarInt)stats.total)); jsvObjectSetChildAndUnLock(o, "freeBytes", jsvNewFromInteger((JsVarInt)stats.free));