Fix inability to handle gaps in the token assignments

sharwell · Sep 29, 2014 · c10b287 · c10b287
1 parent 41e71a7
commit c10b287
Show file tree

Hide file tree

Showing 15 changed files with 17 additions and 16 deletions.
diff --git a/tool/src/main/java/org/antlr/codegen/CodeGenerator.java b/tool/src/main/java/org/antlr/codegen/CodeGenerator.java
@@ -861,10 +861,11 @@ protected void genTokenTypeConstants(ST code) {
 	protected void genTokenTypeNames(ST code) {
 		for (int t=Label.MIN_TOKEN_TYPE; t<=grammar.getMaxTokenType(); t++) {
 			String tokenName = grammar.getTokenDisplayName(t);
-			if ( tokenName!=null ) {
+			if (tokenName != null) {
 				tokenName=target.getTargetStringLiteralFromString(tokenName, true);
-				code.add("tokenNames", tokenName);
 			}
+
+			code.add("tokenNames", tokenName);
 		}
 	}
 

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/ActionScript/ActionScript.stg b/tool/src/main/resources/org/antlr/codegen/templates/ActionScript/ActionScript.stg
@@ -173,7 +173,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 public class <grammar.recognizerName> extends <if(actions.(actionScope).superClass)><actions.(actionScope).superClass><else><@superClassName><superClass><@end><endif> {
 <if(grammar.grammarIsRoot)>
     public static const tokenNames:Array = [
-        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
     ];<\n>
 <endif>
     <tokens:{it |public static const <it.name>:int=<it.type>;}; separator="\n">

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/C/C.stg b/tool/src/main/resources/org/antlr/codegen/templates/C/C.stg
@@ -1195,7 +1195,7 @@ pANTLR3_UINT8   <name>TokenNames[<length(tokenNames)>+4]
         (pANTLR3_UINT8) "\<EOR>",
         (pANTLR3_UINT8) "\<DOWN>",
         (pANTLR3_UINT8) "\<UP>",
-        <tokenNames:{it |(pANTLR3_UINT8) <it>}; separator=",\n">
+        <tokenNames:{it |(pANTLR3_UINT8) <it>}; separator=",\n", null="(pANTLR3_UINT8) \"<invalid>\"">
        };
 <endif>
 

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/CSharp2/CSharp2.stg b/tool/src/main/resources/org/antlr/codegen/templates/CSharp2/CSharp2.stg
@@ -287,7 +287,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 {
 <if(grammar.grammarIsRoot)>
 	internal static readonly string[] tokenNames = new string[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
 	};
 <endif>
 	<tokens:{it|public const int <it.name; format="id">=<it.type>;}; separator="\n">

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/CSharp3/CSharp3.stg b/tool/src/main/resources/org/antlr/codegen/templates/CSharp3/CSharp3.stg
@@ -282,7 +282,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 {
 <if(grammar.grammarIsRoot)>
 	internal static readonly string[] tokenNames = new string[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
 	};
 <endif>
 	<tokens:{it|public const int <it.name; format="id">=<it.type>;}; separator="\n">

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Cpp/Cpp.stg b/tool/src/main/resources/org/antlr/codegen/templates/Cpp/Cpp.stg
@@ -821,7 +821,7 @@ ANTLR_UINT8* <name>TokenNames[<length(tokenNames)>+4]
         (ANTLR_UINT8*) "\<EOR>",
         (ANTLR_UINT8*) "\<DOWN>",
         (ANTLR_UINT8*) "\<UP>",
-        <tokenNames:{it |(ANTLR_UINT8*) <it>}; separator=",\n">
+        <tokenNames:{it |(ANTLR_UINT8*) <it>}; separator=",\n", null="(ANTLR_UINT8*) \"<invalid>\"">
        };
 <endif>
 

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Delphi/Delphi.stg b/tool/src/main/resources/org/antlr/codegen/templates/Delphi/Delphi.stg
@@ -321,7 +321,7 @@ type
         '\<EOR>',
         '\<DOWN>',
         '\<UP>',
-        <tokenNames; separator=",\n">);<\n>
+        <tokenNames; separator=",\n", null="'<invalid>'">);<\n>
 <endif>
   public
     const

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Java/Java.stg b/tool/src/main/resources/org/antlr/codegen/templates/Java/Java.stg
@@ -208,7 +208,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 public class <grammar.recognizerName> extends <@superClassName><superClass><@end> {
 <if(grammar.grammarIsRoot)>
 	public static final String[] tokenNames = new String[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", wrap="\n\t\t">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; null="\"<invalid>\"", separator=", ", wrap="\n\t\t">
 	};
 <endif>
 	<tokens:{it |public static final int <it.name>=<it.type>;}; separator="\n">

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/JavaScript/JavaScript.stg b/tool/src/main/resources/org/antlr/codegen/templates/JavaScript/JavaScript.stg
@@ -217,7 +217,7 @@ org.antlr.lang.augmentObject(<grammar.recognizerName>.prototype, {
 // public class variables
 org.antlr.lang.augmentObject(<grammar.recognizerName>, {
 <if(grammar.grammarIsRoot)>
-    tokenNames: ["\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">],<\n>
+    tokenNames: ["\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">],<\n>
 <endif>
     <bitsets:bitset(name={FOLLOW_<it.name>_in_<it.inName><it.tokenIndex>},
                     words64=it.bits); separator=",\n">

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/ObjC/ObjC.stg b/tool/src/main/resources/org/antlr/codegen/templates/ObjC/ObjC.stg
@@ -712,7 +712,7 @@ genericParser(  grammar,
 {
     #pragma mark Bitsets
     <bitsets:{it | <bitsetInit(name={FOLLOW_<it.name>_in_<it.inName><it.tokenIndex>}, words64=it.bits)>}>
-    [BaseRecognizer setTokenNames:[[AMutableArray arrayWithObjects:@"\<invalid>", @"\<EOR>", @"\<DOWN>", @"\<UP>", <tokenNames:{it | @<it>}; separator=", ", wrap="\n ">, nil] retain]];
+    [BaseRecognizer setTokenNames:[[AMutableArray arrayWithObjects:@"\<invalid>", @"\<EOR>", @"\<DOWN>", @"\<UP>", <tokenNames:{it | @<it>}; separator=", ", null="@\"<invalid>\"", wrap="\n ">, nil] retain]];
     [BaseRecognizer setGrammarFileName:@"<fileName>"];
     <synpreds:{pred | <synpred(pred)>}>
 }

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Perl5/Perl5.stg b/tool/src/main/resources/org/antlr/codegen/templates/Perl5/Perl5.stg
@@ -201,7 +201,7 @@ use Moose;
 extends '<@superClassName><superClass><@end>';
 
 Readonly my $token_names => [
-    "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+    "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
 ];
 
 use constant {

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg b/tool/src/main/resources/org/antlr/codegen/templates/Python/Python.stg
@@ -227,7 +227,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 # token names
 tokenNames = [
     "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>",
-    <tokenNames; wrap, separator=", ">
+    <tokenNames; wrap, separator=", ", null="\"<invalid>\"">
 ]<\n>
 <else>
 from <grammar.composite.rootGrammar.recognizerName> import tokenNames<\n>

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Python3/Python3.stg b/tool/src/main/resources/org/antlr/codegen/templates/Python3/Python3.stg
@@ -229,7 +229,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 # token names
 tokenNames = [
     "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>",
-    <tokenNames; wrap, separator=", ">
+    <tokenNames; wrap, separator=", ", null="\"<invalid>\"">
 ]
 
 <scopes:{it|<if(it.isDynamicGlobalScope)><globalAttributeScopeClass(scope=it)><endif>}>

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Ruby/Ruby.stg b/tool/src/main/resources/org/antlr/codegen/templates/Ruby/Ruby.stg
@@ -118,7 +118,7 @@ module TokenData
   # this is necessary because anonymous tokens, which are
   # created from literal values in the grammar, do not
   # have descriptive names
-  register_names( <tokenNames:{it | <it>}; separator=", ", anchor, wrap="\n"> )
+  register_names( <tokenNames:{it | <it>}; separator=", ", null="\"<invalid>\"", anchor, wrap="\n"> )
 
 <endif>
 

diff --git a/tool/src/main/resources/org/antlr/codegen/templates/Scala/Scala.stg b/tool/src/main/resources/org/antlr/codegen/templates/Scala/Scala.stg
@@ -185,7 +185,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 object <grammar.recognizerName> {
 <if(grammar.grammarIsRoot)>
     val tokenNames = Array(
-        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
     )<\n>
 <endif>