Skip to content

Commit

Permalink
Use new lexer options min/max DFA edge also in lexer ATN simulator
Browse files Browse the repository at this point in the history 
Also make reachesIntoOutContext a boolean as indicated by its TODO item.

Signed-off-by: Mike Lischke <[email protected]>
  • Loading branch information
@mike-lischke
mike-lischke committed Feb 24, 2024
1 parent a268368 commit 247dbc0
Show file tree
Hide file tree
Showing 5 changed files with 25 additions and 71 deletions.
35 changes: 5 additions & 30 deletions src/atn/ATNConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,6 @@ import { Recognizer } from "../Recognizer.js";
import { ATNSimulator } from "./ATNSimulator.js";
import { MurmurHash } from "../utils/MurmurHash.js";

export interface IATNConfigParameters {
state?: ATNState | null,
alt?: number | null,
context?: PredictionContext | null,
semanticContext?: SemanticContext | null,
reachesIntoOuterContext?: number | null,
precedenceFilterSuppressed?: number,
};

export interface ICheckedConfigParameters {
state: ATNState | null,
alt: number | null,
context: PredictionContext | null,
semanticContext: SemanticContext | null,
reachesIntoOuterContext: number | null,
precedenceFilterSuppressed?: boolean,
};

export class ATNConfig {
/** The ATN state associated with this configuration */
public readonly state: ATNState;
Expand All @@ -46,10 +28,9 @@ export class ATNConfig {
* invokes the ATN simulator.
*
* closure() tracks the depth of how far we dip into the outer context:
* depth > 0. Note that it may not be totally accurate depth since I
* don't ever decrement. TODO: make it a boolean then
* depth > 0.
*/
public reachesIntoOuterContext: number = 0; // Not used in hash code.
public reachesIntoOuterContext: boolean = false; // Not used in hash code.

public precedenceFilterSuppressed = false; // Not used in hash code.

Expand Down Expand Up @@ -77,9 +58,7 @@ export class ATNConfig {
this.alt = c.alt!;
this.context = context;
this.#semanticContext = semanticContext ?? SemanticContext.NONE;
if (c.reachesIntoOuterContext !== undefined) {
this.reachesIntoOuterContext = c.reachesIntoOuterContext;
}
this.reachesIntoOuterContext = c.reachesIntoOuterContext!;

if (c.precedenceFilterSuppressed !== undefined) {
this.precedenceFilterSuppressed = c.precedenceFilterSuppressed;
Expand Down Expand Up @@ -157,12 +136,8 @@ export class ATNConfig {

return "(" + this.state + alt +
(this.context !== null ? ",[" + this.context.toString() + "]" : "") +
(this.semanticContext !== SemanticContext.NONE ?
("," + this.semanticContext.toString())
: "") +
(this.reachesIntoOuterContext > 0 ?
(",up=" + this.reachesIntoOuterContext)
: "") + ")";
(this.semanticContext !== SemanticContext.NONE ? ("," + this.semanticContext.toString()) : "") +
(this.reachesIntoOuterContext ? (",up=" + this.reachesIntoOuterContext) : "") + ")";
}

}
37 changes: 11 additions & 26 deletions src/atn/ATNConfigSet.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,16 +126,14 @@ export class ATNConfigSet {

/**
* Adding a new config means merging contexts with existing configs for
* `(s, i, pi, _)`, where `s` is the
* {@link ATNConfig//state}, `i` is the {@link ATNConfig//alt}, and
* `pi` is the {@link ATNConfig//semanticContext}. We use
* `(s,i,pi)` as key.
* `(s, i, pi, _)`, where `s` is the {@link ATNConfig.state}, `i` is the {@link ATNConfig.alt}, and
* `pi` is the {@link ATNConfig.semanticContext}. We use `(s,i,pi)` as key.
*
* This method updates {@link dipsIntoOuterContext} and
* {@link hasSemanticContext} when necessary.
*/
public add(config: ATNConfig,
mergeCache: DoubleDict<PredictionContext, PredictionContext, PredictionContext> | null = null): boolean {
mergeCache: DoubleDict<PredictionContext, PredictionContext, PredictionContext> | null = null): void {
if (this.readOnly) {
throw new Error("This set is readonly");
}
Expand All @@ -144,20 +142,15 @@ export class ATNConfigSet {
this.firstStopState = config;
}

if (config.semanticContext !== SemanticContext.NONE) {
this.hasSemanticContext = true;
}

if (config.reachesIntoOuterContext > 0) {
this.dipsIntoOuterContext = true;
}
this.hasSemanticContext ||= config.semanticContext !== SemanticContext.NONE;
this.dipsIntoOuterContext ||= config.reachesIntoOuterContext;

const existing = this.configLookup!.getOrAdd(config);
if (existing === config) {
this.#cachedHashCode = -1;
this.configs.push(config); // track order here

return true;
return;
}

// a previous (s,i,pi,_), merge with it and save result
Expand All @@ -169,16 +162,12 @@ export class ATNConfigSet {
* since only way to create new graphs is "call rule" and here. We
* cache at both places
*/
existing.reachesIntoOuterContext = Math.max(existing.reachesIntoOuterContext, config.reachesIntoOuterContext);
existing.reachesIntoOuterContext ||= config.reachesIntoOuterContext;

// make sure to preserve the precedence filter suppression during the merge
if (config.precedenceFilterSuppressed) {
existing.precedenceFilterSuppressed = true;
}
existing.precedenceFilterSuppressed ||= config.precedenceFilterSuppressed;

existing.context = merged; // replace context; no need to alt mapping

return true;
}

/** Return a List holding list of configs */
Expand Down Expand Up @@ -261,15 +250,11 @@ export class ATNConfigSet {
}

public hashCode(): number {
if (this.readOnly) {
if (this.#cachedHashCode === -1) {
this.#cachedHashCode = this.computeHashCode();
}

return this.#cachedHashCode;
if (this.#cachedHashCode === -1) {
this.#cachedHashCode = this.computeHashCode();
}

return this.computeHashCode();
return this.#cachedHashCode;
}

public get length(): number {
Expand Down
3 changes: 1 addition & 2 deletions src/atn/AmbiguityInfo.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { DecisionEventInfo } from "./DecisionEventInfo.js";
* determine that the SLL conflict is truly an ambiguity. For example, if none
* of the ATN configurations in the conflicting SLL configuration set have
* traversed a global follow transition (i.e.
* {@link ATNConfig#reachesIntoOuterContext} is 0 for all configurations), then
* {@link ATNConfig.reachesIntoOuterContext} is 0 for all configurations), then
* the result of SLL prediction for that input is known to be equivalent to the
* result of LL prediction for that input.
*
Expand All @@ -32,7 +32,6 @@ import { DecisionEventInfo } from "./DecisionEventInfo.js";
* @see ParserATNSimulator#reportAmbiguity
* @see ANTLRErrorListener#reportAmbiguity
*/

export interface AmbiguityInfo extends DecisionEventInfo {
/** The set of alternative numbers for this decision event that lead to a valid parse. */
ambigAlts: BitSet | null;
Expand Down
19 changes: 7 additions & 12 deletions src/atn/LexerATNSimulator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
/* eslint-disable jsdoc/require-param, jsdoc/require-returns */

import { Token } from "../Token.js";
import { Lexer } from "../Lexer.js";
import { Lexer, type LexerOptions } from "../Lexer.js";
import { ATN } from "./ATN.js";
import { ATNSimulator } from "./ATNSimulator.js";
import { DFAState } from "../dfa/DFAState.js";
Expand Down Expand Up @@ -53,9 +53,6 @@ interface SimState {
}

export class LexerATNSimulator extends ATNSimulator {
// ml: why's that configurable? Memory consumption?
public static readonly MAX_DFA_EDGE = 256; // forces unicode to stay in ATN

public readonly decisionToDFA: DFA[];

public readonly recognizer: Lexer | null = null;
Expand All @@ -79,8 +76,7 @@ export class LexerATNSimulator extends ATNSimulator {
/** Used during DFA/ATN exec to record the most recent accept configuration info */
#prevAccept: SimState | undefined;

#minCodePoint: number;
#maxCodePoint: number;
#options: LexerOptions;

/** Lookup table for lexer ATN config creation. */
#lexerATNConfigFactory: Array<(input: CharStream, config: LexerATNConfig, trans: Transition, configs: ATNConfigSet,
Expand Down Expand Up @@ -109,8 +105,7 @@ export class LexerATNSimulator extends ATNSimulator {
this.recognizer = recog;

if (recog) {
this.#minCodePoint = recog.options.minCodePoint;
this.#maxCodePoint = recog.options.maxCodePoint;
this.#options = recog.options;
}
}

Expand Down Expand Up @@ -261,7 +256,7 @@ export class LexerATNSimulator extends ATNSimulator {
* `t`, or `null` if the target state for this edge is not already cached
*/
private getExistingTargetState(s: DFAState, t: number): DFAState | undefined {
if (t > Token.EOF && t < LexerATNSimulator.MAX_DFA_EDGE) {
if (t >= this.#options.minDFAEdge && t <= this.#options.maxDFAEdge) {
return s.edges[t];
}

Expand Down Expand Up @@ -364,7 +359,7 @@ export class LexerATNSimulator extends ATNSimulator {
}

private getReachableTarget(trans: Transition, t: number): ATNState | undefined {
if (trans.matches(t, this.#minCodePoint, this.#maxCodePoint)) {
if (trans.matches(t, this.#options.minCodePoint, this.#options.maxCodePoint)) {
return trans.target;
} else {
return undefined;
Expand Down Expand Up @@ -537,7 +532,7 @@ export class LexerATNSimulator extends ATNSimulator {
trans: Transition, configs: ATNConfigSet,
speculative: boolean, treatEofAsEpsilon: boolean) => {
if (treatEofAsEpsilon) {
if (trans.matches(Token.EOF, this.#minCodePoint, this.#maxCodePoint)) {
if (trans.matches(Token.EOF, this.#options.minCodePoint, this.#options.maxCodePoint)) {
return LexerATNConfig.createWithConfig(trans.target, config);
}
}
Expand Down Expand Up @@ -630,7 +625,7 @@ export class LexerATNSimulator extends ATNSimulator {
}

// Add the edge.
if (tk === Token.EOF || tk > LexerATNSimulator.MAX_DFA_EDGE) {
if (tk < this.#options.minDFAEdge || tk > this.#options.maxDFAEdge) {
// Only track edges within the DFA bounds
return to!;
}
Expand Down
2 changes: 1 addition & 1 deletion src/atn/ParserATNSimulator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1281,7 +1281,7 @@ export class ParserATNSimulator extends ATNSimulator {
}
}

c.reachesIntoOuterContext = 1;
c.reachesIntoOuterContext = true;
if (closureBusy.getOrAdd(c) !== c) {
// Avoid infinite recursion for right-recursive rules.
continue;
Expand Down

0 comments on commit 247dbc0

Please sign in to comment.