diff --git a/albero/languages/arduino.py b/albero/languages/arduino.py index c31b5d9..77fa167 100644 --- a/albero/languages/arduino.py +++ b/albero/languages/arduino.py @@ -1 +1 @@ -# from tree_sitter_arduino import language as arduino_language (issue opened https://github.com/tree-sitter-grammars/tree-sitter-arduino/issues/30) +from tree_sitter_arduino import language as arduino_language diff --git a/albero/languages/compile_languages.py b/albero/languages/compile_languages.py index d8a6c1e..6917e12 100644 --- a/albero/languages/compile_languages.py +++ b/albero/languages/compile_languages.py @@ -1,6 +1,7 @@ from beartype.typing import Callable from tree_sitter import Language +from .arduino import arduino_language from .bash import bash_language from .c import c_language from .c_sharp import c_sharp_language @@ -31,8 +32,6 @@ from .wgsl_bevy import wgsl_bevy_language from .yaml import yaml_language -# from .arduino import arduino_language - language_functions: dict[str, Callable] = { "starlark": starlark_language, "commonlisp": commonlisp_language, @@ -47,7 +46,7 @@ "php": php_language, "cuda": cuda_language, "pymanifest": pymanifest_language, - # "arduino": arduino_language, + "arduino": arduino_language, "css": css_language, "embedded_template": embedded_template_language, "jsdoc": jsdoc_language, diff --git a/albero/languages/java.py b/albero/languages/java.py index 003bc91..a332197 100644 --- a/albero/languages/java.py +++ b/albero/languages/java.py @@ -3,19 +3,24 @@ java_mapping: dict[str, str] = { "block_comment": "Comment", "line_comment": "Comment", + "abstract": "Keyword", "assert": "Keyword", "boolean": "Keyword", "boolean_type": "Keyword", + "break": "Keyword", "byte": "Keyword", + "case": "Keyword", "catch": "Keyword", "char": "Keyword", "class": "Keyword", "do": "Keyword", + "default": "Keyword", "double": "Keyword", "else": "Keyword", "extends": "Keyword", "false": "Keyword", "final": "Keyword", + "finally": "Keyword", "float": "Keyword", "for": "Keyword", "if": "Keyword", @@ -33,6 +38,7 @@ "return": "Keyword", "short": "Keyword", "static": "Keyword", + "switch": "Keyword", "super": "Keyword", "this": "Keyword", "throw": "Keyword", @@ -41,8 +47,8 @@ "try": "Keyword", "void_type": "Keyword", "while": "Keyword", - "identifier": "Name", - "type_identifier": "Name", + "identifier": "Identifier", + "type_identifier": "Identifier", "decimal_floating_point_literal": "Number", "decimal_integer_literal": "Number", "!": "Operator", @@ -67,6 +73,7 @@ ">=": "Operator", "?": "Operator", "||": "Operator", + "@": "Operator", "(": "Punctuation", ")": "Punctuation", ",": "Punctuation", diff --git a/albero/languages/python.py b/albero/languages/python.py index bae4c5e..ddb7855 100644 --- a/albero/languages/python.py +++ b/albero/languages/python.py @@ -33,8 +33,8 @@ "true": "Keyword", "while": "Keyword", "with": "Keyword", - "@": "Name", - "identifier": "Name", + "@": "Identifier", + "identifier": "Identifier", "float": "Number", "finally": "Number", "integer": "Number", @@ -52,6 +52,8 @@ "/=": "Operator", "<": "Operator", "<=": "Operator", + "%": "Operator", + "%=": "Operator", "=": "Operator", "==": "Operator", ">": "Operator", diff --git a/tests/test-languages/test_java.py b/tests/test-languages/test_java.py index dd8c97e..ab058ea 100644 --- a/tests/test-languages/test_java.py +++ b/tests/test-languages/test_java.py @@ -25,14 +25,14 @@ def test_basic_usage(): assert tokens == [ ((1, 0), 7, "Keyword"), - ((1, 8), 3, "Name"), + ((1, 8), 3, "Identifier"), ((1, 11), 1, "Punctuation"), - ((1, 12), 7, "Name"), + ((1, 12), 7, "Identifier"), ((1, 19), 1, "Punctuation"), ((3, 0), 6, "Keyword"), - ((3, 7), 4, "Name"), + ((3, 7), 4, "Identifier"), ((3, 11), 1, "Punctuation"), - ((3, 12), 4, "Name"), + ((3, 12), 4, "Identifier"), ((3, 16), 1, "Punctuation"), ((3, 17), 1, "Operator"), ((3, 18), 1, "Punctuation"), @@ -42,106 +42,106 @@ def test_basic_usage(): ((7, 1), 2, "Comment"), ((9, 0), 6, "Keyword"), ((9, 7), 5, "Keyword"), - ((9, 13), 4, "Name"), + ((9, 13), 4, "Identifier"), ((9, 18), 1, "Punctuation"), ((11, 4), 6, "Keyword"), ((11, 11), 6, "Keyword"), ((11, 18), 4, "Keyword"), - ((11, 23), 4, "Name"), + ((11, 23), 4, "Identifier"), ((11, 27), 1, "Punctuation"), - ((11, 28), 6, "Name"), + ((11, 28), 6, "Identifier"), ((11, 34), 2, "Punctuation"), - ((11, 37), 4, "Name"), + ((11, 37), 4, "Identifier"), ((11, 41), 1, "Punctuation"), ((11, 43), 1, "Punctuation"), ((12, 8), 15, "Comment"), ((13, 8), 3, "Keyword"), - ((13, 12), 1, "Name"), + ((13, 12), 1, "Identifier"), ((13, 14), 1, "Operator"), ((13, 16), 1, "Number"), ((13, 17), 1, "Punctuation"), ((13, 20), 26, "Comment"), ((14, 8), 6, "Keyword"), - ((14, 15), 1, "Name"), + ((14, 15), 1, "Identifier"), ((14, 17), 1, "Operator"), ((14, 19), 3, "Number"), ((14, 22), 1, "Punctuation"), ((14, 25), 33, "Comment"), ((15, 8), 7, "Keyword"), - ((15, 16), 6, "Name"), + ((15, 16), 6, "Identifier"), ((15, 23), 1, "Operator"), ((15, 25), 4, "Keyword"), ((15, 29), 1, "Punctuation"), ((15, 32), 27, "Comment"), ((16, 8), 7, "Keyword"), - ((16, 16), 7, "Name"), + ((16, 16), 7, "Identifier"), ((16, 24), 1, "Operator"), ((16, 26), 5, "Keyword"), ((16, 31), 1, "Punctuation"), ((16, 34), 28, "Comment"), - ((17, 8), 6, "Name"), - ((17, 15), 3, "Name"), + ((17, 8), 6, "Identifier"), + ((17, 15), 3, "Identifier"), ((17, 19), 1, "Operator"), ((17, 21), 15, "String"), ((17, 36), 1, "Punctuation"), ((17, 39), 9, "Comment"), ((18, 8), 4, "Keyword"), - ((18, 13), 1, "Name"), + ((18, 13), 1, "Identifier"), ((18, 15), 1, "Operator"), ((18, 17), 3, "String"), ((18, 20), 1, "Punctuation"), ((18, 23), 20, "Comment"), ((19, 8), 4, "Keyword"), - ((19, 13), 3, "Name"), + ((19, 13), 3, "Identifier"), ((19, 17), 1, "Operator"), ((19, 19), 4, "String"), ((19, 23), 1, "Punctuation"), ((19, 26), 18, "Comment"), - ((21, 8), 1, "Name"), + ((21, 8), 1, "Identifier"), ((21, 10), 2, "Operator"), ((21, 13), 1, "Number"), ((21, 14), 1, "Punctuation"), ((21, 17), 14, "Comment"), - ((22, 8), 1, "Name"), + ((22, 8), 1, "Identifier"), ((22, 10), 2, "Operator"), ((22, 13), 1, "Number"), ((22, 14), 1, "Punctuation"), ((22, 17), 14, "Comment"), - ((23, 8), 1, "Name"), + ((23, 8), 1, "Identifier"), ((23, 10), 2, "Operator"), ((23, 13), 1, "Number"), ((23, 14), 1, "Punctuation"), ((23, 17), 14, "Comment"), - ((24, 8), 1, "Name"), + ((24, 8), 1, "Identifier"), ((24, 10), 2, "Operator"), ((24, 13), 1, "Number"), ((24, 14), 1, "Punctuation"), ((24, 17), 14, "Comment"), - ((25, 8), 1, "Name"), + ((25, 8), 1, "Identifier"), ((25, 13), 1, "Number"), ((25, 14), 1, "Punctuation"), ((25, 17), 13, "Comment"), - ((26, 8), 1, "Name"), + ((26, 8), 1, "Identifier"), ((26, 9), 2, "Operator"), ((26, 11), 1, "Punctuation"), ((26, 14), 14, "Comment"), - ((27, 8), 1, "Name"), + ((27, 8), 1, "Identifier"), ((27, 9), 2, "Operator"), ((27, 11), 1, "Punctuation"), ((27, 14), 14, "Comment"), ((29, 8), 2, "Keyword"), ((29, 11), 1, "Punctuation"), - ((29, 12), 1, "Name"), + ((29, 12), 1, "Identifier"), ((29, 14), 2, "Operator"), ((29, 17), 1, "Number"), ((29, 18), 1, "Punctuation"), ((29, 20), 1, "Punctuation"), ((29, 23), 21, "Comment"), - ((30, 12), 6, "Name"), + ((30, 12), 6, "Identifier"), ((30, 18), 1, "Punctuation"), - ((30, 19), 3, "Name"), + ((30, 19), 3, "Identifier"), ((30, 22), 1, "Punctuation"), - ((30, 23), 7, "Name"), + ((30, 23), 7, "Identifier"), ((30, 30), 1, "Punctuation"), ((30, 31), 12, "String"), ((30, 43), 2, "Punctuation"), @@ -150,17 +150,17 @@ def test_basic_usage(): ((31, 10), 4, "Keyword"), ((31, 15), 2, "Keyword"), ((31, 18), 1, "Punctuation"), - ((31, 19), 1, "Name"), + ((31, 19), 1, "Identifier"), ((31, 21), 2, "Operator"), ((31, 24), 1, "Number"), ((31, 25), 1, "Punctuation"), ((31, 27), 1, "Punctuation"), ((31, 30), 26, "Comment"), - ((32, 12), 6, "Name"), + ((32, 12), 6, "Identifier"), ((32, 18), 1, "Punctuation"), - ((32, 19), 3, "Name"), + ((32, 19), 3, "Identifier"), ((32, 22), 1, "Punctuation"), - ((32, 23), 7, "Name"), + ((32, 23), 7, "Identifier"), ((32, 30), 1, "Punctuation"), ((32, 31), 16, "String"), ((32, 47), 2, "Punctuation"), @@ -168,24 +168,24 @@ def test_basic_usage(): ((33, 10), 4, "Keyword"), ((33, 15), 1, "Punctuation"), ((33, 18), 15, "Comment"), - ((34, 12), 6, "Name"), + ((34, 12), 6, "Identifier"), ((34, 18), 1, "Punctuation"), - ((34, 19), 3, "Name"), + ((34, 19), 3, "Identifier"), ((34, 22), 1, "Punctuation"), - ((34, 23), 7, "Name"), + ((34, 23), 7, "Identifier"), ((34, 30), 1, "Punctuation"), ((34, 31), 16, "String"), ((34, 47), 2, "Punctuation"), ((35, 8), 1, "Punctuation"), ((37, 8), 5, "Keyword"), ((37, 14), 1, "Punctuation"), - ((37, 15), 1, "Name"), + ((37, 15), 1, "Identifier"), ((37, 17), 1, "Operator"), ((37, 19), 2, "Number"), ((37, 21), 1, "Punctuation"), ((37, 23), 1, "Punctuation"), ((37, 26), 23, "Comment"), - ((38, 12), 1, "Name"), + ((38, 12), 1, "Identifier"), ((38, 13), 2, "Operator"), ((38, 15), 1, "Punctuation"), ((38, 18), 12, "Comment"), @@ -193,26 +193,26 @@ def test_basic_usage(): ((41, 8), 3, "Keyword"), ((41, 12), 1, "Punctuation"), ((41, 13), 3, "Keyword"), - ((41, 17), 1, "Name"), + ((41, 17), 1, "Identifier"), ((41, 19), 1, "Operator"), ((41, 21), 1, "Number"), ((41, 22), 1, "Punctuation"), - ((41, 24), 1, "Name"), + ((41, 24), 1, "Identifier"), ((41, 26), 1, "Operator"), ((41, 28), 2, "Number"), ((41, 30), 1, "Punctuation"), - ((41, 32), 1, "Name"), + ((41, 32), 1, "Identifier"), ((41, 33), 2, "Operator"), ((41, 35), 1, "Punctuation"), ((41, 37), 1, "Punctuation"), ((41, 40), 26, "Comment"), - ((42, 12), 6, "Name"), + ((42, 12), 6, "Identifier"), ((42, 18), 1, "Punctuation"), - ((42, 19), 3, "Name"), + ((42, 19), 3, "Identifier"), ((42, 22), 1, "Punctuation"), - ((42, 23), 7, "Name"), + ((42, 23), 7, "Identifier"), ((42, 30), 1, "Punctuation"), - ((42, 31), 1, "Name"), + ((42, 31), 1, "Identifier"), ((42, 32), 2, "Punctuation"), ((42, 36), 14, "Comment"), ((43, 8), 1, "Punctuation"), @@ -221,7 +221,7 @@ def test_basic_usage(): ((45, 15), 14, "Comment"), ((46, 12), 5, "Keyword"), ((46, 18), 3, "Keyword"), - ((46, 22), 9, "Name"), + ((46, 22), 9, "Identifier"), ((46, 31), 1, "Punctuation"), ((46, 32), 8, "String"), ((46, 40), 2, "Punctuation"), @@ -229,147 +229,156 @@ def test_basic_usage(): ((47, 8), 1, "Punctuation"), ((47, 10), 5, "Keyword"), ((47, 16), 1, "Punctuation"), - ((47, 17), 9, "Name"), - ((47, 27), 1, "Name"), + ((47, 17), 9, "Identifier"), + ((47, 27), 1, "Identifier"), ((47, 28), 1, "Punctuation"), ((47, 30), 1, "Punctuation"), ((47, 33), 16, "Comment"), - ((48, 12), 1, "Name"), + ((48, 12), 1, "Identifier"), ((48, 13), 1, "Punctuation"), - ((48, 14), 15, "Name"), + ((48, 14), 15, "Identifier"), ((48, 29), 3, "Punctuation"), ((48, 34), 14, "Comment"), ((49, 8), 1, "Punctuation"), + ((49, 10), 7, "Keyword"), ((49, 18), 1, "Punctuation"), ((49, 21), 18, "Comment"), - ((50, 12), 6, "Name"), + ((50, 12), 6, "Identifier"), ((50, 18), 1, "Punctuation"), - ((50, 19), 3, "Name"), + ((50, 19), 3, "Identifier"), ((50, 22), 1, "Punctuation"), - ((50, 23), 7, "Name"), + ((50, 23), 7, "Identifier"), ((50, 30), 1, "Punctuation"), ((50, 31), 15, "String"), ((50, 46), 2, "Punctuation"), ((51, 8), 1, "Punctuation"), ((53, 8), 6, "Keyword"), - ((53, 15), 1, "Name"), + ((53, 15), 1, "Identifier"), ((53, 17), 2, "Operator"), ((53, 20), 2, "Number"), ((53, 22), 1, "Punctuation"), ((53, 25), 33, "Comment"), + ((55, 8), 6, "Keyword"), ((55, 15), 1, "Punctuation"), - ((55, 16), 1, "Name"), + ((55, 16), 1, "Identifier"), ((55, 17), 1, "Punctuation"), ((55, 19), 1, "Punctuation"), + ((56, 12), 4, "Keyword"), ((56, 17), 2, "Number"), ((56, 19), 1, "Punctuation"), ((56, 22), 15, "Comment"), - ((57, 16), 6, "Name"), + ((57, 16), 6, "Identifier"), ((57, 22), 1, "Punctuation"), - ((57, 23), 3, "Name"), + ((57, 23), 3, "Identifier"), ((57, 26), 1, "Punctuation"), - ((57, 27), 7, "Name"), + ((57, 27), 7, "Identifier"), ((57, 34), 1, "Punctuation"), ((57, 35), 9, "String"), ((57, 44), 2, "Punctuation"), + ((58, 16), 5, "Keyword"), ((58, 21), 1, "Punctuation"), ((58, 24), 16, "Comment"), + ((59, 12), 7, "Keyword"), ((59, 19), 1, "Punctuation"), - ((60, 16), 6, "Name"), + ((60, 16), 6, "Identifier"), ((60, 22), 1, "Punctuation"), - ((60, 23), 3, "Name"), + ((60, 23), 3, "Identifier"), ((60, 26), 1, "Punctuation"), - ((60, 27), 7, "Name"), + ((60, 27), 7, "Identifier"), ((60, 34), 1, "Punctuation"), ((60, 35), 14, "String"), ((60, 49), 2, "Punctuation"), + ((61, 16), 5, "Keyword"), ((61, 21), 1, "Punctuation"), ((62, 8), 1, "Punctuation"), ((64, 8), 7, "Keyword"), - ((64, 16), 9, "Name"), + ((64, 16), 9, "Identifier"), ((64, 26), 1, "Operator"), - ((64, 28), 1, "Name"), + ((64, 28), 1, "Identifier"), ((64, 30), 1, "Operator"), ((64, 32), 1, "Number"), ((64, 34), 2, "Operator"), - ((64, 37), 1, "Name"), + ((64, 37), 1, "Identifier"), ((64, 39), 1, "Operator"), ((64, 41), 2, "Number"), ((64, 43), 1, "Punctuation"), ((64, 46), 24, "Comment"), ((65, 8), 7, "Keyword"), - ((65, 16), 11, "Name"), + ((65, 16), 11, "Identifier"), ((65, 28), 1, "Operator"), - ((65, 30), 1, "Name"), + ((65, 30), 1, "Identifier"), ((65, 32), 1, "Operator"), ((65, 34), 1, "Number"), ((65, 36), 2, "Operator"), - ((65, 39), 1, "Name"), + ((65, 39), 1, "Identifier"), ((65, 41), 1, "Operator"), ((65, 43), 2, "Number"), ((65, 45), 1, "Punctuation"), ((65, 48), 14, "Comment"), - ((67, 8), 4, "Name"), - ((67, 13), 4, "Name"), + ((67, 8), 4, "Identifier"), + ((67, 13), 4, "Identifier"), ((67, 18), 1, "Operator"), ((67, 20), 3, "Keyword"), - ((67, 24), 4, "Name"), + ((67, 24), 4, "Identifier"), ((67, 28), 3, "Punctuation"), - ((68, 8), 4, "Name"), + ((68, 8), 4, "Identifier"), ((68, 12), 1, "Punctuation"), - ((68, 13), 13, "Name"), + ((68, 13), 13, "Identifier"), ((68, 26), 3, "Punctuation"), ((69, 4), 1, "Punctuation"), ((71, 4), 6, "Keyword"), ((71, 11), 4, "Keyword"), - ((71, 16), 13, "Name"), + ((71, 16), 13, "Identifier"), ((71, 29), 2, "Punctuation"), ((71, 32), 1, "Punctuation"), ((72, 8), 5, "Keyword"), ((72, 13), 1, "Punctuation"), - ((72, 14), 8, "Name"), + ((72, 14), 8, "Identifier"), ((72, 22), 3, "Punctuation"), ((72, 27), 16, "Comment"), ((73, 8), 4, "Keyword"), ((73, 12), 1, "Punctuation"), - ((73, 13), 8, "Name"), + ((73, 13), 8, "Identifier"), ((73, 21), 3, "Punctuation"), ((73, 26), 15, "Comment"), ((74, 4), 1, "Punctuation"), ((76, 4), 6, "Keyword"), ((76, 11), 9, "Keyword"), - ((76, 21), 16, "Name"), + ((76, 21), 16, "Identifier"), ((76, 38), 1, "Punctuation"), ((76, 41), 20, "Comment"), ((77, 8), 4, "Keyword"), - ((77, 13), 7, "Name"), + ((77, 13), 7, "Identifier"), ((77, 20), 3, "Punctuation"), ((77, 25), 22, "Comment"), ((78, 4), 1, "Punctuation"), ((80, 4), 6, "Keyword"), + ((80, 11), 8, "Keyword"), ((80, 20), 5, "Keyword"), - ((80, 26), 20, "Name"), + ((80, 26), 20, "Identifier"), ((80, 47), 1, "Punctuation"), ((80, 50), 17, "Comment"), ((81, 8), 6, "Keyword"), + ((81, 15), 8, "Keyword"), ((81, 24), 4, "Keyword"), - ((81, 29), 7, "Name"), + ((81, 29), 7, "Identifier"), ((81, 36), 3, "Punctuation"), ((81, 41), 18, "Comment"), ((82, 4), 1, "Punctuation"), ((84, 4), 6, "Keyword"), ((84, 11), 5, "Keyword"), - ((84, 17), 12, "Name"), + ((84, 17), 12, "Identifier"), ((84, 30), 7, "Keyword"), - ((84, 38), 20, "Name"), + ((84, 38), 20, "Identifier"), ((84, 59), 10, "Keyword"), - ((84, 70), 16, "Name"), + ((84, 70), 16, "Identifier"), ((84, 87), 1, "Punctuation"), ((84, 90), 38, "Comment"), - ((85, 9), 8, "Name"), + ((85, 8), 1, "Operator"), + ((85, 9), 8, "Identifier"), ((86, 8), 6, "Keyword"), ((86, 15), 4, "Keyword"), - ((86, 20), 7, "Name"), + ((86, 20), 7, "Identifier"), ((86, 27), 2, "Punctuation"), ((86, 30), 1, "Punctuation"), ((86, 33), 23, "Comment"), @@ -379,7 +388,7 @@ def test_basic_usage(): ((91, 4), 7, "Keyword"), ((91, 12), 5, "Keyword"), ((91, 18), 3, "Keyword"), - ((91, 22), 15, "Name"), + ((91, 22), 15, "Identifier"), ((91, 38), 1, "Operator"), ((91, 40), 1, "Number"), ((91, 41), 1, "Punctuation"), @@ -387,21 +396,21 @@ def test_basic_usage(): ((92, 4), 9, "Keyword"), ((92, 14), 6, "Keyword"), ((92, 21), 3, "Keyword"), - ((92, 25), 18, "Name"), + ((92, 25), 18, "Identifier"), ((92, 44), 1, "Operator"), ((92, 46), 1, "Number"), ((92, 47), 1, "Punctuation"), ((92, 50), 29, "Comment"), ((93, 4), 6, "Keyword"), ((93, 11), 3, "Keyword"), - ((93, 15), 9, "Name"), + ((93, 15), 9, "Identifier"), ((93, 25), 1, "Operator"), ((93, 27), 1, "Number"), ((93, 28), 1, "Punctuation"), ((93, 31), 17, "Comment"), ((95, 4), 6, "Keyword"), ((95, 11), 4, "Keyword"), - ((95, 16), 10, "Name"), + ((95, 16), 10, "Identifier"), ((95, 26), 2, "Punctuation"), ((95, 29), 1, "Punctuation"), ((96, 8), 6, "Keyword"), @@ -410,7 +419,7 @@ def test_basic_usage(): ((97, 4), 1, "Punctuation"), ((99, 4), 6, "Keyword"), ((99, 11), 3, "Keyword"), - ((99, 15), 9, "Name"), + ((99, 15), 9, "Identifier"), ((99, 24), 2, "Punctuation"), ((99, 27), 1, "Punctuation"), ((100, 8), 6, "Keyword"), @@ -419,17 +428,17 @@ def test_basic_usage(): ((100, 19), 28, "Comment"), ((101, 4), 1, "Punctuation"), ((103, 4), 7, "Keyword"), - ((103, 12), 12, "Name"), + ((103, 12), 12, "Identifier"), ((103, 25), 1, "Operator"), ((103, 27), 1, "Operator"), - ((103, 28), 6, "Name"), + ((103, 28), 6, "Identifier"), ((103, 34), 1, "Punctuation"), ((103, 37), 13, "Comment"), ((104, 4), 7, "Keyword"), - ((104, 12), 16, "Name"), + ((104, 12), 16, "Identifier"), ((104, 29), 1, "Operator"), ((104, 31), 1, "Punctuation"), - ((104, 32), 1, "Name"), + ((104, 32), 1, "Identifier"), ((104, 34), 2, "Operator"), ((104, 37), 2, "Number"), ((104, 39), 1, "Punctuation"), @@ -441,7 +450,7 @@ def test_basic_usage(): ((104, 58), 23, "Comment"), ((106, 4), 3, "Keyword"), ((106, 7), 2, "Punctuation"), - ((106, 10), 5, "Name"), + ((106, 10), 5, "Identifier"), ((106, 16), 1, "Operator"), ((106, 18), 3, "Keyword"), ((106, 22), 3, "Keyword"), diff --git a/tests/test-languages/test_python.py b/tests/test-languages/test_python.py index a4d9f35..8a79655 100644 --- a/tests/test-languages/test_python.py +++ b/tests/test-languages/test_python.py @@ -25,19 +25,19 @@ def test_basic_usage(): assert tokens == [ ((1, 0), 4, "Keyword"), - ((1, 5), 6, "Name"), + ((1, 5), 6, "Identifier"), ((1, 12), 6, "Keyword"), - ((1, 19), 3, "Name"), + ((1, 19), 3, "Identifier"), ((3, 0), 4, "Keyword"), - ((3, 5), 3, "Name"), + ((3, 5), 3, "Identifier"), ((3, 9), 6, "Keyword"), - ((3, 16), 3, "Name"), + ((3, 16), 3, "Identifier"), ((3, 20), 2, "Keyword"), - ((3, 23), 3, "Name"), + ((3, 23), 3, "Identifier"), ((3, 28), 12, "Comment"), ((5, 0), 17, "Comment"), ((7, 0), 24, "Comment"), - ((8, 0), 1, "Name"), + ((8, 0), 1, "Identifier"), ((8, 2), 1, "Operator"), ((8, 4), 1, "Punctuation"), ((8, 5), 3, "String"), @@ -46,70 +46,70 @@ def test_basic_usage(): ((8, 12), 3, "String"), ((8, 15), 1, "Punctuation"), ((8, 18), 29, "Comment"), - ((9, 0), 1, "Name"), + ((9, 0), 1, "Identifier"), ((9, 2), 1, "Operator"), ((9, 4), 3, "Number"), ((9, 9), 7, "Comment"), - ((10, 0), 1, "Name"), + ((10, 0), 1, "Identifier"), ((10, 2), 1, "Operator"), ((10, 4), 2, "Punctuation"), ((10, 8), 6, "Comment"), - ((11, 0), 1, "Name"), + ((11, 0), 1, "Identifier"), ((11, 2), 1, "Operator"), ((11, 4), 1, "Number"), ((11, 7), 9, "Comment"), - ((12, 0), 1, "Name"), + ((12, 0), 1, "Identifier"), ((12, 2), 1, "Operator"), - ((12, 4), 3, "Name"), + ((12, 4), 3, "Identifier"), ((12, 7), 1, "Punctuation"), ((12, 8), 3, "String"), ((12, 11), 1, "Punctuation"), ((12, 14), 21, "Comment"), - ((13, 0), 1, "Name"), + ((13, 0), 1, "Identifier"), ((13, 2), 1, "Operator"), ((13, 4), 6, "Keyword"), ((13, 10), 1, "Punctuation"), - ((13, 12), 5, "Name"), + ((13, 12), 5, "Identifier"), ((13, 19), 30, "Comment"), ((15, 0), 20, "Comment"), ((16, 0), 2, "Keyword"), - ((16, 3), 1, "Name"), + ((16, 3), 1, "Identifier"), ((16, 5), 2, "Operator"), ((16, 8), 1, "Number"), ((16, 9), 1, "Punctuation"), ((16, 12), 14, "Comment"), - ((17, 4), 1, "Name"), + ((17, 4), 1, "Identifier"), ((17, 6), 2, "Operator"), ((17, 9), 1, "Number"), ((17, 11), 1, "Operator"), - ((17, 13), 1, "Name"), + ((17, 13), 1, "Identifier"), ((17, 16), 34, "Comment"), - ((18, 4), 1, "Name"), + ((18, 4), 1, "Identifier"), ((18, 6), 2, "Operator"), ((18, 9), 1, "Number"), ((18, 11), 1, "Operator"), - ((18, 13), 1, "Name"), + ((18, 13), 1, "Identifier"), ((18, 16), 37, "Comment"), - ((19, 4), 1, "Name"), + ((19, 4), 1, "Identifier"), ((19, 6), 2, "Operator"), ((19, 9), 1, "Number"), ((19, 11), 1, "Operator"), - ((19, 13), 1, "Name"), + ((19, 13), 1, "Identifier"), ((19, 16), 40, "Comment"), - ((20, 4), 1, "Name"), + ((20, 4), 1, "Identifier"), ((20, 6), 2, "Operator"), ((20, 9), 1, "Number"), ((20, 11), 1, "Operator"), - ((20, 13), 1, "Name"), + ((20, 13), 1, "Identifier"), ((20, 16), 34, "Comment"), - ((21, 4), 1, "Name"), + ((21, 4), 1, "Identifier"), ((21, 6), 3, "Operator"), ((21, 10), 1, "Number"), ((21, 11), 2, "Operator"), - ((21, 13), 1, "Name"), + ((21, 13), 1, "Identifier"), ((21, 16), 40, "Comment"), ((22, 4), 2, "Keyword"), - ((22, 7), 1, "Name"), + ((22, 7), 1, "Identifier"), ((22, 9), 2, "Keyword"), ((22, 12), 3, "Keyword"), ((22, 16), 4, "Keyword"), @@ -118,7 +118,7 @@ def test_basic_usage(): ((23, 8), 4, "Keyword"), ((23, 14), 16, "Comment"), ((24, 4), 4, "Keyword"), - ((24, 9), 1, "Name"), + ((24, 9), 1, "Identifier"), ((24, 11), 2, "Keyword"), ((24, 14), 4, "Keyword"), ((24, 18), 1, "Punctuation"), @@ -130,13 +130,13 @@ def test_basic_usage(): ((27, 8), 4, "Keyword"), ((29, 0), 22, "Comment"), ((30, 0), 5, "Keyword"), - ((30, 6), 1, "Name"), + ((30, 6), 1, "Identifier"), ((30, 7), 1, "Punctuation"), ((31, 4), 4, "Keyword"), ((31, 9), 1, "Number"), ((31, 10), 1, "Punctuation"), ((31, 13), 6, "Comment"), - ((32, 8), 5, "Name"), + ((32, 8), 5, "Identifier"), ((32, 13), 1, "Punctuation"), ((32, 14), 6, "String"), ((32, 20), 1, "Punctuation"), @@ -144,7 +144,7 @@ def test_basic_usage(): ((33, 9), 1, "Keyword"), ((33, 10), 1, "Punctuation"), ((33, 13), 8, "Comment"), - ((34, 8), 5, "Name"), + ((34, 8), 5, "Identifier"), ((34, 13), 1, "Punctuation"), ((34, 14), 6, "String"), ((34, 20), 1, "Punctuation"), @@ -152,7 +152,7 @@ def test_basic_usage(): ((37, 0), 5, "Keyword"), ((37, 6), 1, "Number"), ((37, 8), 2, "Keyword"), - ((37, 11), 1, "Name"), + ((37, 11), 1, "Identifier"), ((37, 12), 1, "Punctuation"), ((37, 15), 13, "Comment"), ((38, 4), 5, "Keyword"), @@ -160,77 +160,77 @@ def test_basic_usage(): ((39, 3), 1, "Number"), ((39, 5), 3, "Keyword"), ((39, 9), 2, "Keyword"), - ((39, 12), 1, "Name"), + ((39, 12), 1, "Identifier"), ((39, 13), 1, "Punctuation"), ((39, 16), 17, "Comment"), ((40, 4), 4, "Keyword"), ((43, 0), 25, "Comment"), ((44, 0), 3, "Keyword"), - ((44, 4), 9, "Name"), + ((44, 4), 9, "Identifier"), ((44, 13), 1, "Punctuation"), - ((44, 14), 3, "Name"), + ((44, 14), 3, "Identifier"), ((44, 17), 1, "Punctuation"), ((44, 19), 2, "Operator"), - ((44, 22), 3, "Name"), + ((44, 22), 3, "Identifier"), ((44, 25), 1, "Punctuation"), ((44, 28), 35, "Comment"), ((45, 4), 3, "Keyword"), - ((45, 8), 7, "Name"), + ((45, 8), 7, "Identifier"), ((45, 15), 3, "Punctuation"), ((45, 20), 17, "Comment"), ((46, 8), 6, "Keyword"), - ((46, 15), 3, "Name"), + ((46, 15), 3, "Identifier"), ((46, 18), 2, "Punctuation"), ((46, 22), 18, "Comment"), ((48, 4), 6, "Keyword"), - ((48, 11), 7, "Name"), - ((51, 0), 10, "Name"), + ((48, 11), 7, "Identifier"), + ((51, 0), 10, "Identifier"), ((52, 0), 3, "Keyword"), - ((52, 4), 7, "Name"), + ((52, 4), 7, "Identifier"), ((52, 11), 3, "Punctuation"), ((53, 4), 2, "Keyword"), - ((53, 7), 1, "Name"), + ((53, 7), 1, "Identifier"), ((53, 9), 1, "Operator"), - ((53, 11), 1, "Name"), + ((53, 11), 1, "Identifier"), ((53, 12), 1, "Punctuation"), ((53, 15), 14, "Comment"), ((54, 8), 4, "Keyword"), ((55, 4), 2, "Keyword"), - ((55, 7), 1, "Name"), + ((55, 7), 1, "Identifier"), ((55, 9), 2, "Operator"), - ((55, 12), 1, "Name"), + ((55, 12), 1, "Identifier"), ((55, 13), 1, "Punctuation"), ((55, 16), 23, "Comment"), ((56, 8), 4, "Keyword"), ((57, 4), 2, "Keyword"), - ((57, 7), 1, "Name"), + ((57, 7), 1, "Identifier"), ((57, 9), 1, "Operator"), - ((57, 11), 1, "Name"), + ((57, 11), 1, "Identifier"), ((57, 12), 1, "Punctuation"), ((57, 15), 11, "Comment"), ((58, 8), 4, "Keyword"), ((59, 4), 2, "Keyword"), - ((59, 7), 1, "Name"), + ((59, 7), 1, "Identifier"), ((59, 9), 2, "Operator"), - ((59, 12), 1, "Name"), + ((59, 12), 1, "Identifier"), ((59, 13), 1, "Punctuation"), ((59, 16), 20, "Comment"), ((60, 8), 4, "Keyword"), ((61, 4), 3, "Operator"), ((62, 4), 2, "Keyword"), - ((62, 7), 1, "Name"), + ((62, 7), 1, "Identifier"), ((62, 9), 2, "Operator"), - ((62, 12), 1, "Name"), + ((62, 12), 1, "Identifier"), ((62, 13), 1, "Punctuation"), ((62, 16), 12, "Comment"), ((63, 8), 4, "Keyword"), ((64, 4), 4, "Keyword"), - ((64, 9), 4, "Name"), + ((64, 9), 4, "Identifier"), ((64, 13), 1, "Punctuation"), ((64, 14), 9, "String"), ((64, 23), 1, "Punctuation"), ((64, 25), 2, "Keyword"), - ((64, 28), 1, "Name"), + ((64, 28), 1, "Identifier"), ((64, 29), 1, "Punctuation"), ((64, 32), 43, "Comment"), ((65, 8), 6, "Keyword"), @@ -246,30 +246,30 @@ def test_basic_usage(): ((67, 8), 4, "Keyword"), ((70, 0), 19, "Comment"), ((71, 0), 5, "Keyword"), - ((71, 6), 3, "Name"), + ((71, 6), 3, "Identifier"), ((71, 9), 1, "Punctuation"), ((72, 4), 3, "Keyword"), - ((72, 8), 10, "Name"), + ((72, 8), 10, "Identifier"), ((72, 18), 1, "Punctuation"), - ((72, 19), 4, "Name"), + ((72, 19), 4, "Identifier"), ((72, 23), 2, "Punctuation"), ((73, 8), 5, "Keyword"), - ((73, 14), 9, "Name"), + ((73, 14), 9, "Identifier"), ((73, 23), 1, "Punctuation"), ((73, 24), 4, "String"), ((73, 28), 1, "Punctuation"), ((73, 31), 17, "Comment"), ((76, 0), 33, "Comment"), ((77, 0), 3, "Keyword"), - ((77, 4), 1, "Name"), + ((77, 4), 1, "Identifier"), ((77, 6), 2, "Keyword"), - ((77, 9), 5, "Name"), + ((77, 9), 5, "Identifier"), ((77, 14), 1, "Punctuation"), - ((77, 15), 1, "Name"), + ((77, 15), 1, "Identifier"), ((77, 16), 2, "Punctuation"), ((78, 4), 8, "Keyword"), ((80, 0), 34, "Comment"), - ((81, 0), 1, "Name"), + ((81, 0), 1, "Identifier"), ((81, 2), 1, "Operator"), ((81, 4), 2, "Punctuation"), ((81, 7), 1, "Operator"), diff --git a/tests/test_ipc_highlight.py b/tests/test_ipc_highlight.py index 1f56d4c..f7cdb07 100644 --- a/tests/test_ipc_highlight.py +++ b/tests/test_ipc_highlight.py @@ -28,7 +28,7 @@ def test_basic_usage(): assert tokens == [ ((1, 0), 3, "Keyword"), - ((1, 4), 4, "Name"), + ((1, 4), 4, "Identifier"), ((1, 8), 3, "Punctuation"), ((1, 12), 3, "Operator"), ] diff --git a/tests/test_usage.py b/tests/test_usage.py index 021436e..1b3743a 100644 --- a/tests/test_usage.py +++ b/tests/test_usage.py @@ -31,7 +31,7 @@ def test_basic_usage(): assert tokens == [ ((1, 0), 3, "Keyword"), - ((1, 4), 4, "Name"), + ((1, 4), 4, "Identifier"), ((1, 8), 3, "Punctuation"), ((1, 12), 3, "Operator"), ] @@ -59,7 +59,7 @@ def test_file_removal(): assert tokens == [ ((1, 0), 3, "Keyword"), - ((1, 4), 4, "Name"), + ((1, 4), 4, "Identifier"), ((1, 8), 3, "Punctuation"), ((1, 12), 3, "Operator"), ] @@ -107,19 +107,19 @@ def test_longer_code(): assert tokens == [ ((1, 0), 5, "Keyword"), - ((1, 6), 3, "Name"), + ((1, 6), 3, "Identifier"), ((1, 9), 1, "Punctuation"), ((2, 4), 3, "Keyword"), - ((2, 8), 3, "Name"), + ((2, 8), 3, "Identifier"), ((2, 11), 2, "Punctuation"), ((2, 14), 2, "Operator"), ((2, 17), 4, "Keyword"), ((2, 21), 1, "Punctuation"), ((3, 8), 2, "Keyword"), - ((3, 11), 3, "Name"), + ((3, 11), 3, "Identifier"), ((3, 14), 1, "Punctuation"), ((3, 17), 12, "Comment"), - ((4, 12), 3, "Name"), + ((4, 12), 3, "Identifier"), ((4, 15), 2, "Punctuation"), ((4, 19), 12, "Comment"), ]