From e88a38e4a165911ea8a0e890b906f0dbad0aecee Mon Sep 17 00:00:00 2001 From: doeke Date: Wed, 6 Mar 2024 21:11:36 -0500 Subject: [PATCH 1/9] Add semantic tokens boilerplate --- pylsp/config/schema.json | 7 ++++- pylsp/hookspecs.py | 5 +++ pylsp/plugins/hover.py | 1 + pylsp/plugins/semantic_tokens.py | 52 ++++++++++++++++++++++++++++++++ pylsp/python_lsp.py | 19 ++++++++++++ pyproject.toml | 1 + 6 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 pylsp/plugins/semantic_tokens.py diff --git a/pylsp/config/schema.json b/pylsp/config/schema.json index ba1d36f8..c25684b5 100644 --- a/pylsp/config/schema.json +++ b/pylsp/config/schema.json @@ -232,6 +232,11 @@ "default": true, "description": "Enable or disable the plugin." }, + "pylsp.plugins.semantic_tokens.enabled": { + "type": "boolean", + "default": true, + "description": "Enable or disable the plugin." + }, "pylsp.plugins.jedi_references.enabled": { "type": "boolean", "default": true, @@ -500,4 +505,4 @@ "description": "The name of the folder in which rope stores project configurations and data. Pass `null` for not using such a folder at all." } } -} \ No newline at end of file +} diff --git a/pylsp/hookspecs.py b/pylsp/hookspecs.py index a2549fbc..8bacb89c 100644 --- a/pylsp/hookspecs.py +++ b/pylsp/hookspecs.py @@ -93,6 +93,11 @@ def pylsp_hover(config, workspace, document, position): pass +@hookspec +def pylsp_semantic_tokens(config, workspace, document): + pass + + @hookspec def pylsp_initialize(config, workspace): pass diff --git a/pylsp/plugins/hover.py b/pylsp/plugins/hover.py index ca69d1b3..e926cb57 100644 --- a/pylsp/plugins/hover.py +++ b/pylsp/plugins/hover.py @@ -10,6 +10,7 @@ @hookimpl def pylsp_hover(config, document, position): + log.info("DOEKE! hover.pylsp_hover") code_position = _utils.position_to_jedi_linecolumn(document, position) definitions = document.jedi_script(use_document_path=True).infer(**code_position) word = document.word_at_position(position) diff --git a/pylsp/plugins/semantic_tokens.py b/pylsp/plugins/semantic_tokens.py new file mode 100644 index 00000000..c72aa5f7 --- /dev/null +++ b/pylsp/plugins/semantic_tokens.py @@ -0,0 +1,52 @@ +# Copyright 2017-2020 Palantir Technologies, Inc. +# Copyright 2021- Python Language Server Contributors. + +import logging + +from pylsp import _utils, hookimpl + +log = logging.getLogger(__name__) + + +@hookimpl +def pylsp_semantic_tokens(config, document): + log.info("DOEKE! semantic_tokens.pylsp_semantic_tokens") + return {"data": [0, 2, 5, 0, 0]} + # code_position = _utils.position_to_jedi_linecolumn(document, position) + # definitions = document.jedi_script(use_document_path=True).infer(**code_position) + # word = document.word_at_position(position) + # + # # Find first exact matching definition + # definition = next((x for x in definitions if x.name == word), None) + # + # # Ensure a definition is used if only one is available + # # even if the word doesn't match. An example of this case is 'np' + # # where 'numpy' doesn't match with 'np'. Same for NumPy ufuncs + # if len(definitions) == 1: + # definition = definitions[0] + # + # if not definition: + # return {"contents": ""} + # + # hover_capabilities = config.capabilities.get("textDocument", {}).get("hover", {}) + # supported_markup_kinds = hover_capabilities.get("contentFormat", ["markdown"]) + # preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds) + # + # # Find first exact matching signature + # signature = next( + # ( + # x.to_string() + # for x in definition.get_signatures() + # if (x.name == word and x.type not in ["module"]) + # ), + # "", + # ) + # + # return { + # "contents": _utils.format_docstring( + # # raw docstring returns only doc, without signature + # definition.docstring(raw=True), + # preferred_markup_kind, + # signatures=[signature] if signature else None, + # ) + # } diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index c606a7c6..54f3a77f 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -253,6 +253,8 @@ def _hook(self, hook_name, doc_uri=None, **kwargs): hook_handlers = self.config.plugin_manager.subset_hook_caller( hook_name, self.config.disabled_plugins ) + log.info("DOEKE! disabled %s", self.config.disabled_plugins) + log.info("DOEKE! hook impls %s", hook_handlers._hookimpls) return hook_handlers( config=self.config, workspace=workspace, document=doc, **kwargs ) @@ -276,6 +278,11 @@ def capabilities(self): "commands": flatten(self._hook("pylsp_commands")) }, "hoverProvider": True, + "semanticTokensProvider": { + "legend": {"tokenTypes": ["function"], "tokenModifiers": []}, + "range": False, + "full": True, + }, "referencesProvider": True, "renameProvider": True, "foldingRangeProvider": True, @@ -430,8 +437,15 @@ def highlight(self, doc_uri, position): ) def hover(self, doc_uri, position): + log.info("DOEKE! python_lsp.hover") return self._hook("pylsp_hover", doc_uri, position=position) or {"contents": ""} + def semantic_tokens(self, doc_uri): + log.info("DOEKE! python_lsp.semantic_tokens") + return self._hook("pylsp_semantic_tokens", doc_uri) or { + "data": [1, 0, 10, 0, 0] + } + @_utils.debounce(LINT_DEBOUNCE_S, keyed_by="doc_uri") def lint(self, doc_uri, is_saved): # Since we're debounced, the document may no longer be open @@ -758,8 +772,13 @@ def m_text_document__document_highlight( return self.highlight(textDocument["uri"], position) def m_text_document__hover(self, textDocument=None, position=None, **_kwargs): + log.info("DOEKE! python_lsp.m_text_document__hover") return self.hover(textDocument["uri"], position) + def m_text_document__semantic_tokens__full(self, textDocument=None, **_kwargs): + log.info("DOEKE! python_lsp.m_text_document__semantic_tokens__full") + return self.semantic_tokens(textDocument["uri"]) + def m_text_document__document_symbol(self, textDocument=None, **_kwargs): return self.document_symbols(textDocument["uri"]) diff --git a/pyproject.toml b/pyproject.toml index 4665dcbe..b5de386b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ flake8 = "pylsp.plugins.flake8_lint" jedi_completion = "pylsp.plugins.jedi_completion" jedi_definition = "pylsp.plugins.definition" jedi_hover = "pylsp.plugins.hover" +semantic_tokens = "pylsp.plugins.semantic_tokens" jedi_highlight = "pylsp.plugins.highlight" jedi_references = "pylsp.plugins.references" jedi_rename = "pylsp.plugins.jedi_rename" From 5c9a44ca484706a32ce0c4d70b583ef503ae75ad Mon Sep 17 00:00:00 2001 From: doeke Date: Wed, 6 Mar 2024 23:25:15 -0500 Subject: [PATCH 2/9] First result in the hookspec --- pylsp/hookspecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylsp/hookspecs.py b/pylsp/hookspecs.py index 8bacb89c..37910989 100644 --- a/pylsp/hookspecs.py +++ b/pylsp/hookspecs.py @@ -93,7 +93,7 @@ def pylsp_hover(config, workspace, document, position): pass -@hookspec +@hookspec(firstresult=True) def pylsp_semantic_tokens(config, workspace, document): pass From 26262456052ff1893ba04a8055437dd21d0f894a Mon Sep 17 00:00:00 2001 From: doeke Date: Wed, 6 Mar 2024 23:26:47 -0500 Subject: [PATCH 3/9] Remove the extra logs I don't need anymore --- pylsp/plugins/hover.py | 1 - pylsp/python_lsp.py | 6 ------ 2 files changed, 7 deletions(-) diff --git a/pylsp/plugins/hover.py b/pylsp/plugins/hover.py index e926cb57..ca69d1b3 100644 --- a/pylsp/plugins/hover.py +++ b/pylsp/plugins/hover.py @@ -10,7 +10,6 @@ @hookimpl def pylsp_hover(config, document, position): - log.info("DOEKE! hover.pylsp_hover") code_position = _utils.position_to_jedi_linecolumn(document, position) definitions = document.jedi_script(use_document_path=True).infer(**code_position) word = document.word_at_position(position) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 54f3a77f..9f317ee2 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -253,8 +253,6 @@ def _hook(self, hook_name, doc_uri=None, **kwargs): hook_handlers = self.config.plugin_manager.subset_hook_caller( hook_name, self.config.disabled_plugins ) - log.info("DOEKE! disabled %s", self.config.disabled_plugins) - log.info("DOEKE! hook impls %s", hook_handlers._hookimpls) return hook_handlers( config=self.config, workspace=workspace, document=doc, **kwargs ) @@ -437,11 +435,9 @@ def highlight(self, doc_uri, position): ) def hover(self, doc_uri, position): - log.info("DOEKE! python_lsp.hover") return self._hook("pylsp_hover", doc_uri, position=position) or {"contents": ""} def semantic_tokens(self, doc_uri): - log.info("DOEKE! python_lsp.semantic_tokens") return self._hook("pylsp_semantic_tokens", doc_uri) or { "data": [1, 0, 10, 0, 0] } @@ -772,11 +768,9 @@ def m_text_document__document_highlight( return self.highlight(textDocument["uri"], position) def m_text_document__hover(self, textDocument=None, position=None, **_kwargs): - log.info("DOEKE! python_lsp.m_text_document__hover") return self.hover(textDocument["uri"], position) def m_text_document__semantic_tokens__full(self, textDocument=None, **_kwargs): - log.info("DOEKE! python_lsp.m_text_document__semantic_tokens__full") return self.semantic_tokens(textDocument["uri"]) def m_text_document__document_symbol(self, textDocument=None, **_kwargs): From 7054c187166c3a4f9a243f41068f1d3c828ee0ee Mon Sep 17 00:00:00 2001 From: doeke Date: Thu, 7 Mar 2024 00:30:02 -0500 Subject: [PATCH 4/9] Make the token and modifier kinds --- pylsp/lsp.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/pylsp/lsp.py b/pylsp/lsp.py index 7b3f02ee..dbdc3b90 100644 --- a/pylsp/lsp.py +++ b/pylsp/lsp.py @@ -86,6 +86,50 @@ class SymbolKind: Array = 18 +-1 + + +class SemanticTokenKind: + Namespace = 0 + # represents a generic type. acts as a fallback for types which + # can't be mapped to a specific type like class or enum. + Type = 1 + Class = 2 + Enum = 3 + Interface = 4 + Struct = 5 + TypeParameter = 6 + Parameter = 7 + Variable = 8 + Property = 9 + EnumMember = 10 + Event = 11 + Function = 12 + Method = 13 + Macro = 14 + Keyword = 15 + Modifier = 16 + Comment = 17 + String = 18 + Number = 19 + Regexp = 20 + Operator = 21 + Decorator = 22 # @since 3.17.0 + + +class SemanticTokenModifierKind: + Declaration = 0 + Definition = 1 + Readonly = 2 + Static = 3 + Deprecated = 4 + Abstract = 5 + Async = 6 + Modification = 7 + Documentation = 8 + DefaultLibrary = 9 + + class TextDocumentSyncKind: NONE = 0 FULL = 1 From fc4138067b42bdb5b51c7c4f00bee8db11751ed8 Mon Sep 17 00:00:00 2001 From: doeke Date: Sat, 9 Mar 2024 15:39:18 -0500 Subject: [PATCH 5/9] Put in all token types and enums for them --- pylsp/lsp.py | 81 ++++++++++++++++++++++++--------------------- pylsp/python_lsp.py | 15 ++++++++- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/pylsp/lsp.py b/pylsp/lsp.py index dbdc3b90..48dcf217 100644 --- a/pylsp/lsp.py +++ b/pylsp/lsp.py @@ -6,6 +6,9 @@ https://github.com/Microsoft/language-server-protocol/blob/master/protocol.md """ +from enum import Enum +from typing import NamedTuple + class CompletionItemKind: Text = 1 @@ -86,48 +89,50 @@ class SymbolKind: Array = 18 --1 +class SemanticToken(NamedTuple): + value: int + name: str -class SemanticTokenKind: - Namespace = 0 +class SemanticTokenType(Enum): + Namespace = SemanticToken(0, "namespace") # represents a generic type. acts as a fallback for types which # can't be mapped to a specific type like class or enum. - Type = 1 - Class = 2 - Enum = 3 - Interface = 4 - Struct = 5 - TypeParameter = 6 - Parameter = 7 - Variable = 8 - Property = 9 - EnumMember = 10 - Event = 11 - Function = 12 - Method = 13 - Macro = 14 - Keyword = 15 - Modifier = 16 - Comment = 17 - String = 18 - Number = 19 - Regexp = 20 - Operator = 21 - Decorator = 22 # @since 3.17.0 - - -class SemanticTokenModifierKind: - Declaration = 0 - Definition = 1 - Readonly = 2 - Static = 3 - Deprecated = 4 - Abstract = 5 - Async = 6 - Modification = 7 - Documentation = 8 - DefaultLibrary = 9 + Type = SemanticToken(1, "type") + Class = SemanticToken(2, "class") + Enum = SemanticToken(3, "enum") + Interface = SemanticToken(4, "interface") + Struct = SemanticToken(5, "struct") + TypeParameter = SemanticToken(6, "typeParameter") + Parameter = SemanticToken(7, "parameter") + Variable = SemanticToken(8, "variable") + Property = SemanticToken(9, "property") + EnumMember = SemanticToken(10, "enumMember") + Event = SemanticToken(11, "event") + Function = SemanticToken(12, "function") + Method = SemanticToken(13, "method") + Macro = SemanticToken(14, "macro") + Keyword = SemanticToken(15, "keyword") + Modifier = SemanticToken(16, "modifier") + Comment = SemanticToken(17, "comment") + String = SemanticToken(18, "string") + Number = SemanticToken(19, "number") + Regexp = SemanticToken(20, "regexp") + Operator = SemanticToken(21, "operator") + Decorator = SemanticToken(22, "decorator") # @since 3.17.0 + + +class SemanticTokenModifier(Enum): + Declaration = SemanticToken(0, "declaration") + Definition = SemanticToken(1, "definition") + Readonly = SemanticToken(2, "readonly") + Static = SemanticToken(3, "static") + Deprecated = SemanticToken(4, "deprecated") + Abstract = SemanticToken(5, "abstract") + Async = SemanticToken(6, "async") + Modification = SemanticToken(7, "modification") + Documentation = SemanticToken(8, "documentation") + DefaultLibrary = SemanticToken(9, "defaultLibrary") class TextDocumentSyncKind: diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 9f317ee2..8d365ddf 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -277,7 +277,20 @@ def capabilities(self): }, "hoverProvider": True, "semanticTokensProvider": { - "legend": {"tokenTypes": ["function"], "tokenModifiers": []}, + "legend": { + "tokenTypes": [ + semantic_token_type.value.name + for semantic_token_type in sorted( + lsp.SemanticTokenType, key=lambda x: x.value + ) + ], + "tokenModifiers": [ + semantic_token_modifier.value.name + for semantic_token_modifier in sorted( + lsp.SemanticTokenModifier, key=lambda x: x.value + ) + ], + }, "range": False, "full": True, }, From d7b675ac0275f6f32879ce1d4a436f51ce8c6b7a Mon Sep 17 00:00:00 2001 From: doeke Date: Sat, 9 Mar 2024 17:59:48 -0500 Subject: [PATCH 6/9] Working starting point. Lots to fine-tune but basic augmented highlighting is working! --- pylsp/plugins/semantic_tokens.py | 124 +++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 40 deletions(-) diff --git a/pylsp/plugins/semantic_tokens.py b/pylsp/plugins/semantic_tokens.py index c72aa5f7..756b4d31 100644 --- a/pylsp/plugins/semantic_tokens.py +++ b/pylsp/plugins/semantic_tokens.py @@ -1,52 +1,96 @@ +""" +Cases to consider +- Treesitter highlighting infers class, variable, function by casing and local context. + e.g. if a class is declared as ``class dingus`` and later referenced as + 1. ``dingus`` + 2. ``dingus()`` + then the declaration will be highlighted in the class color, 1. in a variable color, + and 2. in a function color. +- Parameter highlighting. Params in the signature of a function and usage of a function + should be highlighted the same color (turn of hlargs plugin to verify) +- Builtins? +- Dunders? "real" builtin dunders vs something the coder just put dunders around +""" + # Copyright 2017-2020 Palantir Technologies, Inc. # Copyright 2021- Python Language Server Contributors. import logging +from jedi.api.classes import Name + from pylsp import _utils, hookimpl +from pylsp.config.config import Config +from pylsp.lsp import SemanticTokenModifier, SemanticTokenType +from pylsp.workspace import Document log = logging.getLogger(__name__) +# Valid values for type are ``module``, ``class``, ``instance``, ``function``, +# ``param``, ``path``, ``keyword``, ``property`` and ``statement``. +TYPE_MAP = { + "module": SemanticTokenType.Namespace.value.value, + "class": SemanticTokenType.Class.value.value, + # "instance": SemanticTokenType.Type.value.value, + "function": SemanticTokenType.Function.value.value, + "param": SemanticTokenType.Parameter.value.value, + # "path": SemanticTokenType.Type.value.value, + # "keyword": SemanticTokenType.Type.value.value, + "property": SemanticTokenType.Property.value.value, + # "statement": SemanticTokenType.Type.value.value, +} + + +def _semantic_token(d: Name) -> tuple[int, int, int, int, int] | None: + parent_defs = d.goto( + follow_imports=True, + follow_builtin_imports=True, + only_stubs=False, + prefer_stubs=False, + ) + if not parent_defs: + log.info("DOEKE! NO PARENT DEFS") + return None + if len(parent_defs) > 1: + log.info("DOEKE! MORE THAN ONE PARENT DEF") + parent_def, *_ = parent_defs + log.info("DOEKE! PARENT DEF DESCRIPTION %s", parent_def.description) + log.info("DOEKE! PARENT DEF TYPE %s", parent_def.type) + if (parent_type := TYPE_MAP.get(parent_def.type, None)) is None: + return None + # if d.name == "self": + # modifier = 2**SemanticTokenModifier.Readonly.value.value + # else: + # modifier = 0 + return (d.line - 1, d.column, len(d.name), parent_type, 0) + @hookimpl -def pylsp_semantic_tokens(config, document): +def pylsp_semantic_tokens(config: Config, document: Document): log.info("DOEKE! semantic_tokens.pylsp_semantic_tokens") - return {"data": [0, 2, 5, 0, 0]} - # code_position = _utils.position_to_jedi_linecolumn(document, position) - # definitions = document.jedi_script(use_document_path=True).infer(**code_position) - # word = document.word_at_position(position) - # - # # Find first exact matching definition - # definition = next((x for x in definitions if x.name == word), None) - # - # # Ensure a definition is used if only one is available - # # even if the word doesn't match. An example of this case is 'np' - # # where 'numpy' doesn't match with 'np'. Same for NumPy ufuncs - # if len(definitions) == 1: - # definition = definitions[0] - # - # if not definition: - # return {"contents": ""} - # - # hover_capabilities = config.capabilities.get("textDocument", {}).get("hover", {}) - # supported_markup_kinds = hover_capabilities.get("contentFormat", ["markdown"]) - # preferred_markup_kind = _utils.choose_markup_kind(supported_markup_kinds) - # - # # Find first exact matching signature - # signature = next( - # ( - # x.to_string() - # for x in definition.get_signatures() - # if (x.name == word and x.type not in ["module"]) - # ), - # "", - # ) - # - # return { - # "contents": _utils.format_docstring( - # # raw docstring returns only doc, without signature - # definition.docstring(raw=True), - # preferred_markup_kind, - # signatures=[signature] if signature else None, - # ) - # } + log.info("DOEKE! %s %s", type(config), type(document)) + + symbols_settings = config.plugin_settings("semantic_tokens") + definitions = document.jedi_names( + all_scopes=True, definitions=True, references=True + ) + data = [] + line, start_char = 0, 0 + for d in definitions: + log.info("DOEKE! %s (%s) %s:%s", d.description, d.type, d.line, d.column) + raw_token = _semantic_token(d) + # log.info("DOEKE! raw token %s", raw_token) + if raw_token is None: + continue + t_line, t_start_char, t_length, t_type, t_mod = raw_token + delta_start_char, start_char = ( + (t_start_char - start_char, t_start_char) + if t_line == line + else (t_start_char, t_start_char) + ) + delta_line, line = t_line - line, t_line + new_token = [delta_line, delta_start_char, t_length, t_type, t_mod] + log.info("DOEKE! diff token %s", new_token) + data.extend(new_token) + + return {"data": data} From 93e707d942dbc2fbaf41f0bbda6e1a3901c0c066 Mon Sep 17 00:00:00 2001 From: doeke Date: Sun, 10 Mar 2024 00:16:47 -0500 Subject: [PATCH 7/9] Change a few names --- pylsp/plugins/semantic_tokens.py | 46 +++++++++++++++----------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/pylsp/plugins/semantic_tokens.py b/pylsp/plugins/semantic_tokens.py index 756b4d31..a6ca0b69 100644 --- a/pylsp/plugins/semantic_tokens.py +++ b/pylsp/plugins/semantic_tokens.py @@ -19,7 +19,7 @@ from jedi.api.classes import Name -from pylsp import _utils, hookimpl +from pylsp import hookimpl from pylsp.config.config import Config from pylsp.lsp import SemanticTokenModifier, SemanticTokenType from pylsp.workspace import Document @@ -35,50 +35,48 @@ "function": SemanticTokenType.Function.value.value, "param": SemanticTokenType.Parameter.value.value, # "path": SemanticTokenType.Type.value.value, - # "keyword": SemanticTokenType.Type.value.value, + "keyword": SemanticTokenType.Keyword.value.value, "property": SemanticTokenType.Property.value.value, - # "statement": SemanticTokenType.Type.value.value, + # "statement": SemanticTokenType.Variable.value.value, } -def _semantic_token(d: Name) -> tuple[int, int, int, int, int] | None: - parent_defs = d.goto( +def _semantic_token(n: Name) -> tuple[int, int, int, int, int] | None: + definitions = n.goto( follow_imports=True, follow_builtin_imports=True, only_stubs=False, prefer_stubs=False, ) - if not parent_defs: - log.info("DOEKE! NO PARENT DEFS") + if not definitions: + log.info("DOEKE! no definitions") return None - if len(parent_defs) > 1: - log.info("DOEKE! MORE THAN ONE PARENT DEF") - parent_def, *_ = parent_defs - log.info("DOEKE! PARENT DEF DESCRIPTION %s", parent_def.description) - log.info("DOEKE! PARENT DEF TYPE %s", parent_def.type) - if (parent_type := TYPE_MAP.get(parent_def.type, None)) is None: + if len(definitions) > 1: + log.info("DOEKE! more than one definition") + definition, *_ = definitions + log.info("DOEKE! definition type: %s", definition.type) + if (definition_type := TYPE_MAP.get(definition.type, None)) is None: return None # if d.name == "self": - # modifier = 2**SemanticTokenModifier.Readonly.value.value + # modifier = ( + # 2**SemanticTokenModifier.Readonly.value.value + # + 2**SemanticTokenModifier.DefaultLibrary.value.value + # ) # else: # modifier = 0 - return (d.line - 1, d.column, len(d.name), parent_type, 0) + return (n.line - 1, n.column, len(n.name), definition_type, 0) @hookimpl def pylsp_semantic_tokens(config: Config, document: Document): - log.info("DOEKE! semantic_tokens.pylsp_semantic_tokens") - log.info("DOEKE! %s %s", type(config), type(document)) - symbols_settings = config.plugin_settings("semantic_tokens") - definitions = document.jedi_names( - all_scopes=True, definitions=True, references=True - ) + names = document.jedi_names(all_scopes=True, definitions=True, references=True) data = [] line, start_char = 0, 0 - for d in definitions: - log.info("DOEKE! %s (%s) %s:%s", d.description, d.type, d.line, d.column) - raw_token = _semantic_token(d) + for n in names: + log.info("DOEKE! name: %s, (%s:%s)", n.name, n.line, n.column) + log.info("DOEKE! type: %s", n.type) + raw_token = _semantic_token(n) # log.info("DOEKE! raw token %s", raw_token) if raw_token is None: continue From d374347d3be4744b3871e6b40e892b2e626363a7 Mon Sep 17 00:00:00 2001 From: doeke Date: Sun, 10 Mar 2024 17:35:47 -0400 Subject: [PATCH 8/9] Refactor a few things, improve log messages, add some docstrings --- pylsp/plugins/semantic_tokens.py | 102 ++++++++++++++++++------------- 1 file changed, 61 insertions(+), 41 deletions(-) diff --git a/pylsp/plugins/semantic_tokens.py b/pylsp/plugins/semantic_tokens.py index a6ca0b69..fb8d4a21 100644 --- a/pylsp/plugins/semantic_tokens.py +++ b/pylsp/plugins/semantic_tokens.py @@ -1,17 +1,3 @@ -""" -Cases to consider -- Treesitter highlighting infers class, variable, function by casing and local context. - e.g. if a class is declared as ``class dingus`` and later referenced as - 1. ``dingus`` - 2. ``dingus()`` - then the declaration will be highlighted in the class color, 1. in a variable color, - and 2. in a function color. -- Parameter highlighting. Params in the signature of a function and usage of a function - should be highlighted the same color (turn of hlargs plugin to verify) -- Builtins? -- Dunders? "real" builtin dunders vs something the coder just put dunders around -""" - # Copyright 2017-2020 Palantir Technologies, Inc. # Copyright 2021- Python Language Server Contributors. @@ -21,7 +7,7 @@ from pylsp import hookimpl from pylsp.config.config import Config -from pylsp.lsp import SemanticTokenModifier, SemanticTokenType +from pylsp.lsp import SemanticTokenType from pylsp.workspace import Document log = logging.getLogger(__name__) @@ -41,7 +27,20 @@ } -def _semantic_token(n: Name) -> tuple[int, int, int, int, int] | None: +def _raw_semantic_token(n: Name) -> list[int] | None: + """Find an appropriate semantic token for the name. + + This works by looking up the definition (using jedi ``goto``) of the name and + matching the definition's type to one of the availabile semantic tokens. Further + improvements are possible by inspecting context, e.g. semantic token modifiers such + as ``abstract`` or ``async`` or even different tokens, e.g. ``property`` or + ``method``. Dunder methods may warrant special treatment/modifiers as well. + + The return is a "raw" semantic token rather than a "diff." This is in the form of a + length 5 array of integers where the elements are the line number, starting + character, length, token index, and modifiers (as an integer whose binary + representation has bits set at the indices of all applicable modifiers). + """ definitions = n.goto( follow_imports=True, follow_builtin_imports=True, @@ -49,46 +48,67 @@ def _semantic_token(n: Name) -> tuple[int, int, int, int, int] | None: prefer_stubs=False, ) if not definitions: - log.info("DOEKE! no definitions") + log.debug( + "no definitions found for name %s (%s:%s)", n.description, n.line, n.column + ) return None if len(definitions) > 1: - log.info("DOEKE! more than one definition") + log.debug( + "multiple definitions found for name %s (%s:%s)", + n.description, + n.line, + n.column, + ) definition, *_ = definitions - log.info("DOEKE! definition type: %s", definition.type) if (definition_type := TYPE_MAP.get(definition.type, None)) is None: + log.debug( + "no matching semantic token for name %s (%s:%s)", + n.description, + n.line, + n.column, + ) return None - # if d.name == "self": - # modifier = ( - # 2**SemanticTokenModifier.Readonly.value.value - # + 2**SemanticTokenModifier.DefaultLibrary.value.value - # ) - # else: - # modifier = 0 - return (n.line - 1, n.column, len(n.name), definition_type, 0) + return [n.line - 1, n.column, len(n.name), definition_type, 0] + + +def _diff_position( + token_line: int, token_start_char: int, current_line: int, current_start_char: int +) -> tuple[int, int, int, int]: + """Compute the diff position for a semantic token. + + This returns the delta line and column as well as what should be considered the + "new" current line and column. + """ + delta_start_char = ( + token_start_char - current_start_char + if token_line == current_line + else token_start_char + ) + delta_line = token_line - current_line + return (delta_line, delta_start_char, token_line, token_start_char) @hookimpl def pylsp_semantic_tokens(config: Config, document: Document): + # Currently unused, but leaving it here for easy adding of settings. symbols_settings = config.plugin_settings("semantic_tokens") + names = document.jedi_names(all_scopes=True, definitions=True, references=True) data = [] line, start_char = 0, 0 for n in names: - log.info("DOEKE! name: %s, (%s:%s)", n.name, n.line, n.column) - log.info("DOEKE! type: %s", n.type) - raw_token = _semantic_token(n) - # log.info("DOEKE! raw token %s", raw_token) - if raw_token is None: + token = _raw_semantic_token(n) + log.debug( + "raw token for name %s (%s:%s): %s", n.description, n.line, n.column, token + ) + if token is None: continue - t_line, t_start_char, t_length, t_type, t_mod = raw_token - delta_start_char, start_char = ( - (t_start_char - start_char, t_start_char) - if t_line == line - else (t_start_char, t_start_char) + token[0], token[1], line, start_char = _diff_position( + token[0], token[1], line, start_char + ) + log.debug( + "diff token for name %s (%s:%s): %s", n.description, n.line, n.column, token ) - delta_line, line = t_line - line, t_line - new_token = [delta_line, delta_start_char, t_length, t_type, t_mod] - log.info("DOEKE! diff token %s", new_token) - data.extend(new_token) + data.extend(token) return {"data": data} From b3425f2b6fab2cc60b020be7ead70e9ba79561a0 Mon Sep 17 00:00:00 2001 From: doeke Date: Sun, 10 Mar 2024 17:39:51 -0400 Subject: [PATCH 9/9] Remove dummy data --- pylsp/python_lsp.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pylsp/python_lsp.py b/pylsp/python_lsp.py index 8d365ddf..a527fe47 100644 --- a/pylsp/python_lsp.py +++ b/pylsp/python_lsp.py @@ -451,9 +451,7 @@ def hover(self, doc_uri, position): return self._hook("pylsp_hover", doc_uri, position=position) or {"contents": ""} def semantic_tokens(self, doc_uri): - return self._hook("pylsp_semantic_tokens", doc_uri) or { - "data": [1, 0, 10, 0, 0] - } + return self._hook("pylsp_semantic_tokens", doc_uri) or {"data": []} @_utils.debounce(LINT_DEBOUNCE_S, keyed_by="doc_uri") def lint(self, doc_uri, is_saved):