Add support for escaped identifiers

This commit mirrors https://github.com/segmentio/fql/pull/53

One slight difference is that `fql` stores the escaped representation of
the identifier internally but this implementation stores the raw
representation of the identifier. `fql` behaves differently because it
proxies the identifier to the `gjson` library during evaluation, which
needs the escaped version, whereas `fql-ts` proxies to and from the raw
UI, where users will not expect to see the escaped version of an
identifier.

src/lexer.test.ts

@@ -39,14 +39,15 @@ test('Lexer passes error fixtures', () => {
     fix('5.0.', [], true),
     fix('5.0.0.0', [], true),
     fix('!', [], true),
+    fix('abc\\', [], true),
+    fix('abc/', [], true),
 
     // Invalid characters
     fix('$', [], true),
     fix('@', [], true),
     fix('#', [], true),
     fix('*', [], true),
     fix('`', [], true),
-    fix('\\', [], true),
     fix('/', [], true)
   ])
 })
@@ -76,7 +77,10 @@ test('Lexer passes ident fixtures', () => {
 
     // Dangerous idents for stupid javascript reasons
     fix('Infinity', [t.Ident('Infinity'), t.EOS()], false),
-    fix('undefined', [t.Ident('undefined'), t.EOS()], false)
+    fix('undefined', [t.Ident('undefined'), t.EOS()], false),
+
+    // Escaped idents
+    fix('a\\ b\\$c\\.', [t.Ident('a b$c.'), t.EOS()], false),
   ])
 })
 

src/lexer.ts

@@ -228,7 +228,16 @@ export class Lexer {
   private lexIdent(previous: string): Token {
     let str = ''
     while (isIdent(this.peek())) {
-      const { char } = this.next()
+      var { char } = this.next()
+
+      // Allow escaping of any character except EOS
+      if (char == '\\') {
+        if (this.peek() == EOS_FLAG) {
+          throw new LexerError('expected character after escape character, got EOS', this.cursor)
+        }
+        char = this.next().char
+      }
+
       str += char
 
       if (str.length >= MAXIMUM_INDENT_LENGTH) {

src/strings.ts

@@ -25,7 +25,7 @@ export function isIdent(c: string): boolean {
     return false
   }
 
-  return isAlpha(c) || isNumber(c) || c === '_' || c === '-'
+  return isAlpha(c) || isNumber(c) || c === '_' || c === '-' || c === '\\'
 }
 
 export function isTerminator(c: string): boolean {

src/unlexer.test.ts

@@ -8,6 +8,12 @@ test('Unlexer can convert tokens into a string', () => {
   expect(str).toEqual('message = 20')
 })
 
+test('Unlexer escapes', () => {
+  const str = unlex([t.Ident('a \\ b $')]).code
+
+  expect(str).toEqual('a\\ \\\\\\ b\\ \\$')
+})
+
 test('Unlexer and lexer play nicely together', () => {
   const code = 'message = 20'
   expect(unlex(lex(code).tokens).code).toBe(code)

src/unlexer.ts

@@ -1,4 +1,5 @@
 import { Token, TokenType } from './token'
+import { isIdent } from './strings'
 
 export class UnlexerError extends Error {
   constructor(public message: string) {
@@ -42,8 +43,23 @@ export default function unlex(tokens: Token[]): UnLexResponse {
       continue
     }
 
-    str += ' ' + token.value
+    if (token.type === TokenType.Ident) {
+      str += ' ' + escape(token.value)
+    } else {
+      str += ' ' + token.value
+    }
   }
 
   return { code: str.trim() }
 }
+
+function escape(str: string): string {
+  let escaped = ''
+  for (const c of str) {
+    if (!isIdent(c) || c == '\\') {
+      escaped += '\\'
+    }
+    escaped += c
+  }
+  return escaped
+}