diff --git a/package.json b/package.json index ddd5bab..be37b47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@segment/fql", - "version": "1.6.1", + "version": "1.7.0", "main": "dist/index.js", "browser": "dist/index.js", "bin": "dist/index.js", diff --git a/src/lexer.test.ts b/src/lexer.test.ts index f24aa15..38b567c 100644 --- a/src/lexer.test.ts +++ b/src/lexer.test.ts @@ -39,6 +39,8 @@ test('Lexer passes error fixtures', () => { fix('5.0.', [], true), fix('5.0.0.0', [], true), fix('!', [], true), + fix('abc\\', [], true), + fix('abc/', [], true), // Invalid characters fix('$', [], true), @@ -46,7 +48,6 @@ test('Lexer passes error fixtures', () => { fix('#', [], true), fix('*', [], true), fix('`', [], true), - fix('\\', [], true), fix('/', [], true) ]) }) @@ -76,7 +77,10 @@ test('Lexer passes ident fixtures', () => { // Dangerous idents for stupid javascript reasons fix('Infinity', [t.Ident('Infinity'), t.EOS()], false), - fix('undefined', [t.Ident('undefined'), t.EOS()], false) + fix('undefined', [t.Ident('undefined'), t.EOS()], false), + + // Escaped idents + fix('a\\ b\\$c\\.', [t.Ident('a b$c.'), t.EOS()], false), ]) }) diff --git a/src/lexer.ts b/src/lexer.ts index 0ab726c..0e3cabc 100644 --- a/src/lexer.ts +++ b/src/lexer.ts @@ -228,7 +228,16 @@ export class Lexer { private lexIdent(previous: string): Token { let str = '' while (isIdent(this.peek())) { - const { char } = this.next() + let { char } = this.next() + + // Allow escaping of any character except EOS + if (char == '\\') { + if (this.peek() == EOS_FLAG) { + throw new LexerError('expected character after escape character, got EOS', this.cursor) + } + char = this.next().char + } + str += char if (str.length >= MAXIMUM_INDENT_LENGTH) { diff --git a/src/strings.ts b/src/strings.ts index 37855c7..868694a 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -25,7 +25,7 @@ export function isIdent(c: string): boolean { return false } - return isAlpha(c) || isNumber(c) || c === '_' || c === '-' + return isAlpha(c) || isNumber(c) || c === '_' || c === '-' || c === '\\' } export function isTerminator(c: string): boolean { diff --git a/src/unlexer.test.ts b/src/unlexer.test.ts index e72f8d8..5181bf0 100644 --- a/src/unlexer.test.ts +++ b/src/unlexer.test.ts @@ -8,6 +8,12 @@ test('Unlexer can convert tokens into a string', () => { expect(str).toEqual('message = 20') }) +test('Unlexer escapes', () => { + const str = unlex([t.Ident('a \\ b $')]).code + + expect(str).toEqual('a\\ \\\\\\ b\\ \\$') +}) + test('Unlexer and lexer play nicely together', () => { const code = 'message = 20' expect(unlex(lex(code).tokens).code).toBe(code) diff --git a/src/unlexer.ts b/src/unlexer.ts index d885d2c..ac4754e 100644 --- a/src/unlexer.ts +++ b/src/unlexer.ts @@ -1,4 +1,5 @@ import { Token, TokenType } from './token' +import { isIdent } from './strings' export class UnlexerError extends Error { constructor(public message: string) { @@ -42,8 +43,23 @@ export default function unlex(tokens: Token[]): UnLexResponse { continue } - str += ' ' + token.value + if (token.type === TokenType.Ident) { + str += ' ' + escape(token.value) + } else { + str += ' ' + token.value + } } return { code: str.trim() } } + +function escape(str: string): string { + let escaped = '' + for (const c of str) { + if (!isIdent(c) || c == '\\') { + escaped += '\\' + } + escaped += c + } + return escaped +} \ No newline at end of file