From cb038e7a2c61a5422a33a14ef873a83218bdb436 Mon Sep 17 00:00:00 2001
From: Tyson Mote <tyson@segment.com>
Date: Thu, 19 Sep 2019 12:10:28 -0700
Subject: [PATCH 1/3] Add support for escaped identifiers

This commit mirrors https://github.com/segmentio/fql/pull/53

One slight difference is that `fql` stores the escaped representation of
the identifier internally but this implementation stores the raw
representation of the identifier. `fql` behaves differently because it
proxies the identifier to the `gjson` library during evaluation, which
needs the escaped version, whereas `fql-ts` proxies to and from the raw
UI, where users will not expect to see the escaped version of an
identifier.
---
 src/lexer.test.ts   |  8 ++++++--
 src/lexer.ts        | 11 ++++++++++-
 src/strings.ts      |  2 +-
 src/unlexer.test.ts |  6 ++++++
 src/unlexer.ts      | 18 +++++++++++++++++-
 5 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/src/lexer.test.ts b/src/lexer.test.ts
index f24aa15..38b567c 100644
--- a/src/lexer.test.ts
+++ b/src/lexer.test.ts
@@ -39,6 +39,8 @@ test('Lexer passes error fixtures', () => {
     fix('5.0.', [], true),
     fix('5.0.0.0', [], true),
     fix('!', [], true),
+    fix('abc\\', [], true),
+    fix('abc/', [], true),
 
     // Invalid characters
     fix('$', [], true),
@@ -46,7 +48,6 @@ test('Lexer passes error fixtures', () => {
     fix('#', [], true),
     fix('*', [], true),
     fix('`', [], true),
-    fix('\\', [], true),
     fix('/', [], true)
   ])
 })
@@ -76,7 +77,10 @@ test('Lexer passes ident fixtures', () => {
 
     // Dangerous idents for stupid javascript reasons
     fix('Infinity', [t.Ident('Infinity'), t.EOS()], false),
-    fix('undefined', [t.Ident('undefined'), t.EOS()], false)
+    fix('undefined', [t.Ident('undefined'), t.EOS()], false),
+
+    // Escaped idents
+    fix('a\\ b\\$c\\.', [t.Ident('a b$c.'), t.EOS()], false),
   ])
 })
 
diff --git a/src/lexer.ts b/src/lexer.ts
index 0ab726c..5014e35 100644
--- a/src/lexer.ts
+++ b/src/lexer.ts
@@ -228,7 +228,16 @@ export class Lexer {
   private lexIdent(previous: string): Token {
     let str = ''
     while (isIdent(this.peek())) {
-      const { char } = this.next()
+      var { char } = this.next()
+
+      // Allow escaping of any character except EOS
+      if (char == '\\') {
+        if (this.peek() == EOS_FLAG) {
+          throw new LexerError('expected character after escape character, got EOS', this.cursor)
+        }
+        char = this.next().char
+      }
+
       str += char
 
       if (str.length >= MAXIMUM_INDENT_LENGTH) {
diff --git a/src/strings.ts b/src/strings.ts
index 37855c7..868694a 100644
--- a/src/strings.ts
+++ b/src/strings.ts
@@ -25,7 +25,7 @@ export function isIdent(c: string): boolean {
     return false
   }
 
-  return isAlpha(c) || isNumber(c) || c === '_' || c === '-'
+  return isAlpha(c) || isNumber(c) || c === '_' || c === '-' || c === '\\'
 }
 
 export function isTerminator(c: string): boolean {
diff --git a/src/unlexer.test.ts b/src/unlexer.test.ts
index e72f8d8..5181bf0 100644
--- a/src/unlexer.test.ts
+++ b/src/unlexer.test.ts
@@ -8,6 +8,12 @@ test('Unlexer can convert tokens into a string', () => {
   expect(str).toEqual('message = 20')
 })
 
+test('Unlexer escapes', () => {
+  const str = unlex([t.Ident('a \\ b $')]).code
+
+  expect(str).toEqual('a\\ \\\\\\ b\\ \\$')
+})
+
 test('Unlexer and lexer play nicely together', () => {
   const code = 'message = 20'
   expect(unlex(lex(code).tokens).code).toBe(code)
diff --git a/src/unlexer.ts b/src/unlexer.ts
index d885d2c..ac4754e 100644
--- a/src/unlexer.ts
+++ b/src/unlexer.ts
@@ -1,4 +1,5 @@
 import { Token, TokenType } from './token'
+import { isIdent } from './strings'
 
 export class UnlexerError extends Error {
   constructor(public message: string) {
@@ -42,8 +43,23 @@ export default function unlex(tokens: Token[]): UnLexResponse {
       continue
     }
 
-    str += ' ' + token.value
+    if (token.type === TokenType.Ident) {
+      str += ' ' + escape(token.value)
+    } else {
+      str += ' ' + token.value
+    }
   }
 
   return { code: str.trim() }
 }
+
+function escape(str: string): string {
+  let escaped = ''
+  for (const c of str) {
+    if (!isIdent(c) || c == '\\') {
+      escaped += '\\'
+    }
+    escaped += c
+  }
+  return escaped
+}
\ No newline at end of file

From 8d8268d96033bd29e994f4e05073ab52379b6cd1 Mon Sep 17 00:00:00 2001
From: Tyson Mote <tyson@segment.com>
Date: Thu, 19 Sep 2019 12:32:00 -0700
Subject: [PATCH 2/3] Use let instead of var

---
 src/lexer.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lexer.ts b/src/lexer.ts
index 5014e35..0e3cabc 100644
--- a/src/lexer.ts
+++ b/src/lexer.ts
@@ -228,7 +228,7 @@ export class Lexer {
   private lexIdent(previous: string): Token {
     let str = ''
     while (isIdent(this.peek())) {
-      var { char } = this.next()
+      let { char } = this.next()
 
       // Allow escaping of any character except EOS
       if (char == '\\') {

From f7019b0a94c83750ea639f1b12dce05a588e21a3 Mon Sep 17 00:00:00 2001
From: Tyson Mote <tyson@segment.com>
Date: Thu, 19 Sep 2019 12:33:56 -0700
Subject: [PATCH 3/3] Bump version to 1.7.0

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index ddd5bab..be37b47 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@segment/fql",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "main": "dist/index.js",
   "browser": "dist/index.js",
   "bin": "dist/index.js",