Bug fixes in code which is rarely used (and therefore had no tests)

- Found and fixed an issue with left recursive rules. - Added a serialization/deserialization unit test. - Fixed missing ambiguity info in parse info (note: tests still pending for that). Signed-off-by: Mike Lischke <[email protected]>
mike-lischke · Nov 20, 2024 · 9daf2ea · 9daf2ea
1 parent 213870f
commit 9daf2ea
Show file tree

Hide file tree

Showing 10 changed files with 94 additions and 43 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -7,24 +7,19 @@
         {
             "type": "node",
             "request": "launch",
-            "name": "Run current Jest test",
-            "runtimeExecutable": "node",
-            "runtimeArgs": [
-                "--experimental-vm-modules",
-                "${workspaceRoot}/node_modules/.bin/jest",
-                "${relativeFile}",
-                "--no-coverage",
-                "--runInBand"
+            "name": "Debug Current Test File",
+            "program": "${workspaceFolder}/node_modules/vitest/vitest.mjs",
+            "args": [
+                "run",
+                "${relativeFile}"
             ],
-            "console": "integratedTerminal",
-            "stopOnEntry": false,
-            "sourceMaps": true,
-            "resolveSourceMapLocations": [
-                "${workspaceFolder}/**",
-                "!**/node_modules/**"
+            "autoAttachChildProcesses": true,
+            "skipFiles": [
+                "<node_internals>/**",
+                "**/node_modules/**"
             ],
-            "smartStep": true,
-            "trace": false
+            "console": "integratedTerminal",
+            "smartStep": true
         },
         {
             "type": "node",

diff --git a/src/Parser.ts b/src/Parser.ts
@@ -27,6 +27,7 @@ import type { IntStream } from "./IntStream.js";
 import type { ParseTreePattern } from "./tree/pattern/ParseTreePattern.js";
 import { Lexer } from "./Lexer.js";
 import { ParseTreePatternMatcher } from "./tree/pattern/ParseTreePatternMatcher.js";
+import { ParseInfo } from "./atn/ParseInfo.js";
 
 export interface IDebugPrinter {
     println(s: string): void;
@@ -706,6 +707,15 @@ export abstract class Parser extends Recognizer<ParserATNSimulator> {
         return this.inputStream.getSourceName();
     }
 
+    public override getParseInfo(): ParseInfo | undefined {
+        const interp = this.interpreter;
+        if (interp instanceof ProfilingATNSimulator) {
+            return new ParseInfo(interp);
+        }
+
+        return undefined;
+    }
+
     public setProfile(profile: boolean): void {
         const interp = this.interpreter;
         const saveMode = interp.predictionMode;

diff --git a/src/ParserInterpreter.ts b/src/ParserInterpreter.ts
@@ -108,7 +108,7 @@ export class ParserInterpreter extends Parser {
         const startRuleStartState = this.#atn.ruleToStartState[startRuleIndex]!;
 
         this.rootContext = this.createInterpreterRuleContext(null, ATNState.INVALID_STATE_NUMBER, startRuleIndex);
-        if (startRuleStartState.isPrecedenceRule) {
+        if (startRuleStartState.isLeftRecursiveRule) {
             this.enterRecursionRule(this.rootContext, startRuleStartState.stateNumber, startRuleIndex, 0);
         }
         else {
@@ -120,8 +120,8 @@ export class ParserInterpreter extends Parser {
             switch ((p.constructor as typeof ATNState).stateType) {
                 case ATNState.RULE_STOP:
                     // pop; return from rule
-                    if (this.context?.isEmpty) {
-                        if (startRuleStartState.isPrecedenceRule) {
+                    if (this.context?.isEmpty()) {
+                        if (startRuleStartState.isLeftRecursiveRule) {
                             const result = this.context;
                             const parentContext = this.parentContextStack.pop()!;
                             this.unrollRecursionContexts(parentContext[0]);
@@ -218,7 +218,7 @@ export class ParserInterpreter extends Parser {
                 const ruleStartState = transition.target as RuleStartState;
                 const ruleIndex = ruleStartState.ruleIndex;
                 const newContext = this.createInterpreterRuleContext(this.context, p.stateNumber, ruleIndex);
-                if (ruleStartState.isPrecedenceRule) {
+                if (ruleStartState.isLeftRecursiveRule) {
                     this.enterRecursionRule(newContext, ruleStartState.stateNumber, ruleIndex,
                         (transition as RuleTransition).precedence);
                 }
@@ -279,7 +279,7 @@ export class ParserInterpreter extends Parser {
 
     protected visitRuleStopState(p: ATNState): void {
         const ruleStartState = this.#atn.ruleToStartState[p.ruleIndex]!;
-        if (ruleStartState.isPrecedenceRule) {
+        if (ruleStartState.isLeftRecursiveRule) {
             const [parentContext, state] = this.parentContextStack.pop()!;
             this.unrollRecursionContexts(parentContext);
             this.state = state;

diff --git a/src/Recognizer.ts b/src/Recognizer.ts
@@ -158,8 +158,8 @@ export abstract class Recognizer<ATNInterpreter extends ATNSimulator> {
         throw new Error("there is no serialized ATN");
     }
 
-    public getParseInfo(): ParseInfo | null {
-        return null;
+    public getParseInfo(): ParseInfo | undefined {
+        return undefined;
     }
 
     // TODO: remove need for this: public abstract get literalNames(): Array<string | null>;

diff --git a/src/atn/ATNDeserializer.ts b/src/atn/ATNDeserializer.ts
@@ -182,7 +182,7 @@ export class ATNDeserializer {
         const numPrecedenceStates = this.data[this.pos++];
         for (j = 0; j < numPrecedenceStates; j++) {
             stateNumber = this.data[this.pos++];
-            (atn.states[stateNumber] as RuleStartState).isPrecedenceRule = true;
+            (atn.states[stateNumber] as RuleStartState).isLeftRecursiveRule = true;
         }
     }
 
@@ -267,7 +267,7 @@ export class ATNDeserializer {
                     continue;
                 }
                 let outermostPrecedenceReturn = -1;
-                if (atn.ruleToStartState[t.target.ruleIndex]!.isPrecedenceRule) {
+                if (atn.ruleToStartState[t.target.ruleIndex]!.isLeftRecursiveRule) {
                     if (t.precedence === 0) {
                         outermostPrecedenceReturn = t.target.ruleIndex;
                     }
@@ -363,7 +363,7 @@ export class ATNDeserializer {
         let excludeTransition = null;
         let endState = null;
 
-        if (atn.ruleToStartState[idx]!.isPrecedenceRule) {
+        if (atn.ruleToStartState[idx]!.isLeftRecursiveRule) {
             // wrap from the beginning of the rule to the StarLoopEntryState
             endState = null;
             for (i = 0; i < atn.states.length; i++) {
@@ -448,7 +448,7 @@ export class ATNDeserializer {
             // We analyze the ATN to determine if this ATN decision state is the
             // decision for the closure block that determines whether a
             // precedence rule should continue or complete.
-            if (atn.ruleToStartState[state.ruleIndex]!.isPrecedenceRule) {
+            if (atn.ruleToStartState[state.ruleIndex]!.isLeftRecursiveRule) {
                 const maybeLoopEndState = state.transitions[state.transitions.length - 1].target;
                 if (maybeLoopEndState instanceof LoopEndState) {
                     if (maybeLoopEndState.epsilonOnlyTransitions &&

diff --git a/src/atn/ATNSerializer.ts b/src/atn/ATNSerializer.ts
@@ -6,6 +6,7 @@
 /* eslint-disable jsdoc/require-returns */
 
 import { Token } from "../Token.js";
+import { HashMap } from "../misc/HashMap.js";
 import type { IntervalSet } from "../misc/IntervalSet.js";
 import { ObjectEqualityComparator } from "../misc/ObjectEqualityComparator.js";
 import { OrderedHashMap } from "../misc/OrderedHashMap.js";
@@ -61,14 +62,15 @@ export class ATNSerializer {
 
         for (const set of sets) {
             const containsEof = set.contains(Token.EOF);
-            if (containsEof && set.get(0).stop === Token.EOF) {
-                data.push(set.length - 1);
+            const intervals = [...set];
+            if (containsEof && intervals[0].stop === Token.EOF) {
+                data.push(intervals.length - 1);
             } else {
-                data.push(set.length);
+                data.push(intervals.length);
             }
 
             data.push(containsEof ? 1 : 0);
-            for (const interval of set) {
+            for (const interval of intervals) {
                 if (interval.start === Token.EOF) {
                     if (interval.stop === Token.EOF) {
                         continue;
@@ -118,8 +120,7 @@ export class ATNSerializer {
         this.addRuleStatesAndLexerTokenTypes();
         this.addModeStartStates();
 
-        let setIndices = null;
-        setIndices = this.addSets();
+        const setIndices = this.addSets();
         this.addEdges(edgeCount, setIndices);
         this.addDecisionStartStates();
         this.addLexerActions();
@@ -212,13 +213,13 @@ export class ATNSerializer {
     }
 
     private addEdges(): number;
-    private addEdges(edgeCount: number, setIndices: Map<IntervalSet, number>): void;
+    private addEdges(edgeCount: number, setIndices: HashMap<IntervalSet, number>): void;
     private addEdges(...args: unknown[]): number | void {
         switch (args.length) {
             case 0: {
-
                 let edgeCount = 0;
                 this.data.push(this.atn.states.length);
+                let i = 0;
                 for (const s of this.atn.states) {
                     if (s === null) { // might be optimized away
                         this.data.push(ATNState.INVALID_TYPE);
@@ -230,12 +231,15 @@ export class ATNSerializer {
                         this.nonGreedyStates.push(s.stateNumber);
                     }
 
+                    if (i === 910) {
+                        console.log("i", i);
+                    }
+
                     if (s instanceof RuleStartState && s.isLeftRecursiveRule) {
                         this.precedenceStates.push(s.stateNumber);
                     }
 
                     this.data.push(stateType);
-
                     this.data.push(s.ruleIndex);
 
                     if ((s.constructor as typeof ATNState).stateType === ATNState.LOOP_END) {
@@ -258,13 +262,14 @@ export class ATNSerializer {
                             this.sets.set(st.set, true);
                         }
                     }
+                    ++i;
                 }
 
                 return edgeCount;
             }
 
             case 2: {
-                const [edgeCount, setIndices] = args as [number, Map<IntervalSet, number>];
+                const [edgeCount, setIndices] = args as [number, HashMap<IntervalSet, number>];
 
                 this.data.push(edgeCount);
                 for (const s of this.atn.states) {
@@ -374,9 +379,9 @@ export class ATNSerializer {
         }
     }
 
-    private addSets(): Map<IntervalSet, number> {
+    private addSets(): HashMap<IntervalSet, number> {
         ATNSerializer.serializeSets(this.data, [...this.sets.keys()]);
-        const setIndices = new Map<IntervalSet, number>();
+        const setIndices = new HashMap<IntervalSet, number>();
         let setIndex = 0;
         for (const s of this.sets.keys()) {
             setIndices.set(s, setIndex++);

diff --git a/src/atn/ParserATNSimulator.ts b/src/atn/ParserATNSimulator.ts
@@ -778,7 +778,7 @@ export class ParserATNSimulator extends ATNSimulator {
         // the fact that we should predict alternative 1.  We just can't say for
         // sure that there is an ambiguity without looking further.
 
-        this.reportAmbiguity(dfa, D, startIndex, input.index, foundExactAmbig, undefined, reach);
+        this.reportAmbiguity(dfa, D, startIndex, input.index, foundExactAmbig, reach.getAlts(), reach);
 
         return predictedAlt;
     }

diff --git a/src/atn/RuleStartState.ts b/src/atn/RuleStartState.ts
@@ -12,5 +12,4 @@ export class RuleStartState extends ATNState {
 
     public stopState?: RuleStopState;
     public isLeftRecursiveRule: boolean = false;
-    public isPrecedenceRule: boolean = false;
 }
diff --git a/src/misc/BitSet.ts b/src/misc/BitSet.ts
@@ -158,8 +158,7 @@ export class BitSet implements Iterable<number> {
 
         // Iterate over all set bits.
         for (const index of this) {
-            // Use the first index > than the specified value index.
-            if (index > fromIndex) {
+            if (index >= fromIndex) {
                 return index;
             }
         }

diff --git a/tests/api/Serialization.spec.ts b/tests/api/Serialization.spec.ts
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+import { describe, expect, it } from "vitest";
+
+import { ATNDeserializer, ATNSerializer } from "../../src/index.js";
+import { MySQLLexer } from "../benchmarks/generated/MySQLLexer.js";
+import { MySQLParser } from "../benchmarks/generated/MySQLParser.js";
+
+describe("Test Serialization and Deserialization", () => {
+    it("Lexer", () => {
+        // eslint-disable-next-line no-underscore-dangle
+        const serialized = MySQLLexer._serializedATN;
+
+        const deserializer = new ATNDeserializer();
+        const atn = deserializer.deserialize(serialized);
+
+        const serializer = new ATNSerializer(atn);
+        const serialized2 = serializer.serialize();
+
+        expect(serialized2.length).toBe(serialized.length);
+
+        expect(serialized2).toEqual(serialized);
+    });
+
+    it("Parser", () => {
+        // eslint-disable-next-line no-underscore-dangle
+        const serialized = MySQLParser._serializedATN;
+
+        const deserializer = new ATNDeserializer();
+        const atn = deserializer.deserialize(serialized);
+
+        const serializer = new ATNSerializer(atn);
+        const serialized2 = serializer.serialize();
+
+        expect(serialized2.length).toBe(serialized.length);
+
+        expect(serialized2).toEqual(serialized);
+    });
+});