Merge pull request #59 from 1c-syntax/feature/eofHidden

EOF in hidden channel
1c-syntax · Apr 15, 2020 · 47f797e · 47f797e
2 parents 71be3f1 + 96231d3
commit 47f797e
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java b/src/main/java/com/github/_1c_syntax/bsl/parser/Tokenizer.java
@@ -25,6 +25,7 @@
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.CodePointCharStream;
+import org.antlr.v4.runtime.CommonToken;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ConsoleErrorListener;
 import org.antlr.v4.runtime.Lexer;
@@ -75,8 +76,8 @@ private List<Token> computeTokens() {
     List<Token> tokensTemp = new ArrayList<>(getTokenStream().getTokens());
 
     Token lastToken = tokensTemp.get(tokensTemp.size() - 1);
-    if (lastToken.getType() == EOF) {
-      tokensTemp.remove(tokensTemp.size() - 1);
+    if (lastToken.getType() == EOF && lastToken instanceof CommonToken) {
+      ((CommonToken)lastToken).setChannel(Lexer.HIDDEN);
     }
 
     return tokensTemp;

diff --git a/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java b/src/test/java/com/github/_1c_syntax/bsl/parser/TokenizerTest.java
@@ -21,6 +21,7 @@
  */
 package com.github._1c_syntax.bsl.parser;
 
+import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.junit.jupiter.api.Test;
@@ -40,7 +41,9 @@ void computeTokens() {
         final List<Token> tokens = tokenizer.getTokens();
 
         // then
-        assertThat(tokens).hasSize(9);
+        assertThat(tokens).hasSize(10);
+        assertThat(tokens.get(9).getType()).isEqualTo(Lexer.EOF);
+        assertThat(tokens.get(9).getChannel()).isEqualTo(Lexer.HIDDEN);
     }
 
     @Test