diff --git a/pyls/_utils.py b/pyls/_utils.py index e5ecef9c..fa7d6cd5 100644 --- a/pyls/_utils.py +++ b/pyls/_utils.py @@ -1,4 +1,5 @@ # Copyright 2017 Palantir Technologies, Inc. +from distutils.version import LooseVersion import functools import inspect import logging @@ -7,7 +8,10 @@ import threading import re +import jedi + PY2 = sys.version_info.major == 2 +JEDI_VERSION = jedi.__version__ if PY2: import pathlib2 as pathlib @@ -137,6 +141,8 @@ def format_docstring(contents): """ contents = contents.replace('\t', u'\u00A0' * 4) contents = contents.replace(' ', u'\u00A0' * 2) + if LooseVersion(JEDI_VERSION) < LooseVersion('0.15.0'): + contents = contents.replace('*', '\\*') return contents diff --git a/pyls/hookspecs.py b/pyls/hookspecs.py index 880be1ff..a52f9902 100644 --- a/pyls/hookspecs.py +++ b/pyls/hookspecs.py @@ -67,6 +67,11 @@ def pyls_experimental_capabilities(config, workspace): pass +@hookspec(firstresult=True) +def pyls_folding_range(config, workspace, document): + pass + + @hookspec(firstresult=True) def pyls_format_document(config, workspace, document): pass diff --git a/pyls/plugins/definition.py b/pyls/plugins/definition.py index 69109afd..8ec3b1ad 100644 --- a/pyls/plugins/definition.py +++ b/pyls/plugins/definition.py @@ -20,6 +20,14 @@ def pyls_definitions(config, document, position): 'end': {'line': d.line - 1, 'character': d.column + len(d.name)}, } } - for d in definitions - if d.is_definition() and d.line is not None and d.column is not None and d.module_path is not None + for d in definitions if d.is_definition() and _not_internal_definition(d) ] + + +def _not_internal_definition(definition): + return ( + definition.line is not None and + definition.column is not None and + definition.module_path is not None and + not definition.in_builtin_module() + ) diff --git a/pyls/plugins/folding.py b/pyls/plugins/folding.py new file mode 100644 index 00000000..48cfdfd5 --- /dev/null +++ b/pyls/plugins/folding.py @@ -0,0 +1,169 @@ +# pylint: disable=len-as-condition +# Copyright 2019 Palantir Technologies, Inc. + +import re + +import parso +import parso.python.tree as tree_nodes + +from pyls import hookimpl + +SKIP_NODES = (tree_nodes.Module, tree_nodes.IfStmt, tree_nodes.TryStmt) +IDENTATION_REGEX = re.compile(r'(\s+).+') + + +@hookimpl +def pyls_folding_range(document): + program = document.source + '\n' + lines = program.splitlines() + tree = parso.parse(program) + ranges = __compute_folding_ranges(tree, lines) + + results = [] + for (start_line, end_line) in ranges: + start_line -= 1 + end_line -= 1 + # If start/end character is not defined, then it defaults to the + # corresponding line last character + results.append({ + 'startLine': start_line, + 'endLine': end_line, + }) + return results + + +def __merge_folding_ranges(left, right): + for start in list(left.keys()): + right_start = right.pop(start, None) + if right_start is not None: + left[start] = max(right_start, start) + left.update(right) + return left + + +def __empty_identation_stack(identation_stack, level_limits, + current_line, folding_ranges): + while identation_stack != []: + upper_level = identation_stack.pop(0) + level_start = level_limits.pop(upper_level) + folding_ranges.append((level_start, current_line)) + return folding_ranges + + +def __match_identation_stack(identation_stack, level, level_limits, + folding_ranges, current_line): + upper_level = identation_stack.pop(0) + while upper_level >= level: + level_start = level_limits.pop(upper_level) + folding_ranges.append((level_start, current_line)) + upper_level = identation_stack.pop(0) + identation_stack.insert(0, upper_level) + return identation_stack, folding_ranges + + +def __compute_folding_ranges_identation(text): + lines = text.splitlines() + folding_ranges = [] + identation_stack = [] + level_limits = {} + current_level = 0 + current_line = 0 + while lines[current_line] == '': + current_line += 1 + for i, line in enumerate(lines): + if i < current_line: + continue + i += 1 + identation_match = IDENTATION_REGEX.match(line) + if identation_match is not None: + whitespace = identation_match.group(1) + level = len(whitespace) + if level > current_level: + level_limits[current_level] = current_line + identation_stack.insert(0, current_level) + current_level = level + elif level < current_level: + identation_stack, folding_ranges = __match_identation_stack( + identation_stack, level, level_limits, folding_ranges, + current_line) + current_level = level + else: + folding_ranges = __empty_identation_stack( + identation_stack, level_limits, current_line, folding_ranges) + current_level = 0 + if line.strip() != '': + current_line = i + folding_ranges = __empty_identation_stack( + identation_stack, level_limits, current_line, folding_ranges) + return dict(folding_ranges) + + +def __check_if_node_is_valid(node): + valid = True + if isinstance(node, tree_nodes.PythonNode): + kind = node.type + valid = kind not in {'decorated', 'parameters'} + if kind == 'suite': + if isinstance(node.parent, tree_nodes.Function): + valid = False + return valid + + +def __compute_start_end_lines(node, stack): + start_line, _ = node.start_pos + end_line, _ = node.end_pos + + last_leaf = node.get_last_leaf() + last_newline = isinstance(last_leaf, tree_nodes.Newline) + last_operator = isinstance(last_leaf, tree_nodes.Operator) + node_is_operator = isinstance(node, tree_nodes.Operator) + last_operator = last_operator or not node_is_operator + + end_line -= 1 + + modified = False + if isinstance(node.parent, tree_nodes.PythonNode): + kind = node.type + if kind in {'suite', 'atom', 'atom_expr', 'arglist'}: + if len(stack) > 0: + next_node = stack[0] + next_line, _ = next_node.start_pos + if next_line > end_line: + end_line += 1 + modified = True + if not last_newline and not modified and not last_operator: + end_line += 1 + return start_line, end_line + + +def __compute_folding_ranges(tree, lines): + folding_ranges = {} + stack = [tree] + + while len(stack) > 0: + node = stack.pop(0) + if isinstance(node, tree_nodes.Newline): + # Skip newline nodes + continue + elif isinstance(node, tree_nodes.PythonErrorNode): + # Fallback to identation-based (best-effort) folding + start_line, _ = node.start_pos + start_line -= 1 + padding = [''] * start_line + text = '\n'.join(padding + lines[start_line:]) + '\n' + identation_ranges = __compute_folding_ranges_identation(text) + folding_ranges = __merge_folding_ranges( + folding_ranges, identation_ranges) + break + elif not isinstance(node, SKIP_NODES): + valid = __check_if_node_is_valid(node) + if valid: + start_line, end_line = __compute_start_end_lines(node, stack) + if end_line > start_line: + current_end = folding_ranges.get(start_line, -1) + folding_ranges[start_line] = max(current_end, end_line) + if hasattr(node, 'children'): + stack = node.children + stack + + folding_ranges = sorted(folding_ranges.items()) + return folding_ranges diff --git a/pyls/plugins/hover.py b/pyls/plugins/hover.py index 86f80c31..1ac57bf5 100644 --- a/pyls/plugins/hover.py +++ b/pyls/plugins/hover.py @@ -1,5 +1,7 @@ # Copyright 2017 Palantir Technologies, Inc. +from distutils.version import LooseVersion import logging + from pyls import hookimpl, _utils log = logging.getLogger(__name__) @@ -10,26 +12,34 @@ def pyls_hover(document, position): definitions = document.jedi_script(position).goto_definitions() word = document.word_at_position(position) - # Find first exact matching definition - definition = next((x for x in definitions if x.name == word), None) - - if not definition: - return {'contents': ''} + if LooseVersion(_utils.JEDI_VERSION) >= LooseVersion('0.15.0'): + # Find first exact matching definition + definition = next((x for x in definitions if x.name == word), None) + + if not definition: + return {'contents': ''} + + # raw docstring returns only doc, without signature + doc = _utils.format_docstring(definition.docstring(raw=True)) + + # Find first exact matching signature + signature = next((x.to_string() for x in definition.get_signatures() if x.name == word), '') + + contents = [] + if signature: + contents.append({ + 'language': 'python', + 'value': signature, + }) + if doc: + contents.append(doc) + if not contents: + return {'contents': ''} + return {'contents': contents} + else: + # Find an exact match for a completion + for d in definitions: + if d.name == word: + return {'contents': _utils.format_docstring(d.docstring()) or ''} - # raw docstring returns only doc, without signature - doc = _utils.format_docstring(definition.docstring(raw=True)) - - # Find first exact matching signature - signature = next((x.to_string() for x in definition.get_signatures() if x.name == word), '') - - contents = [] - if signature: - contents.append({ - 'language': 'python', - 'value': signature, - }) - if doc: - contents.append(doc) - if not contents: return {'contents': ''} - return {'contents': contents} diff --git a/pyls/python_ls.py b/pyls/python_ls.py index 586b5b89..9084bc77 100644 --- a/pyls/python_ls.py +++ b/pyls/python_ls.py @@ -1,8 +1,9 @@ # Copyright 2017 Palantir Technologies, Inc. +from functools import partial import logging +import os import socketserver import threading -from functools import partial from pyls_jsonrpc.dispatchers import MethodDispatcher from pyls_jsonrpc.endpoint import Endpoint @@ -33,7 +34,16 @@ def setup(self): self.delegate = self.DELEGATE_CLASS(self.rfile, self.wfile) def handle(self): - self.delegate.start() + try: + self.delegate.start() + except OSError as e: + if os.name == 'nt': + # Catch and pass on ConnectionResetError when parent process + # dies + # pylint: disable=no-member, undefined-variable + if isinstance(e, WindowsError) and e.winerror == 10054: + pass + # pylint: disable=no-member self.SHUTDOWN_CALL() @@ -163,6 +173,7 @@ def capabilities(self): 'hoverProvider': True, 'referencesProvider': True, 'renameProvider': True, + 'foldingRangeProvider': True, 'signatureHelpProvider': { 'triggerCharacters': ['(', ',', '='] }, @@ -202,7 +213,7 @@ def m_initialize(self, processId=None, rootUri=None, rootPath=None, initializati def watch_parent_process(pid): # exit when the given pid is not alive if not _utils.is_process_alive(pid): - log.info("parent process %s is not alive", pid) + log.info("parent process %s is not alive, exiting!", pid) self.m_exit() else: threading.Timer(PARENT_PROCESS_WATCH_INTERVAL, watch_parent_process, args=[pid]).start() @@ -272,6 +283,9 @@ def rename(self, doc_uri, position, new_name): def signature_help(self, doc_uri, position): return self._hook('pyls_signature_help', doc_uri, position=position) + def folding(self, doc_uri): + return self._hook('pyls_folding_range', doc_uri) + def m_text_document__did_close(self, textDocument=None, **_kwargs): workspace = self._match_uri_to_workspace(textDocument['uri']) workspace.rm_document(textDocument['uri']) @@ -323,6 +337,9 @@ def m_text_document__formatting(self, textDocument=None, _options=None, **_kwarg def m_text_document__rename(self, textDocument=None, position=None, newName=None, **_kwargs): return self.rename(textDocument['uri'], position, newName) + def m_text_document__folding_range(self, textDocument=None, **_kwargs): + return self.folding(textDocument['uri']) + def m_text_document__range_formatting(self, textDocument=None, range=None, _options=None, **_kwargs): # Again, we'll ignore formatting options for now. return self.format_range(textDocument['uri'], range) diff --git a/setup.py b/setup.py index 2b9c7d46..12ac7ed4 100755 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ 'configparser; python_version<"3.0"', 'future>=0.14.0; python_version<"3"', 'backports.functools_lru_cache; python_version<"3.2"', - 'jedi>=0.15.0,<0.16', + 'jedi>=0.14.1,<0.16', 'python-jsonrpc-server>=0.1.0', 'pluggy' ], @@ -67,7 +67,8 @@ 'pylint': ['pylint'], 'rope': ['rope>0.10.5'], 'yapf': ['yapf'], - 'test': ['versioneer', 'pylint', 'pytest', 'mock', 'pytest-cov', 'coverage'], + 'test': ['versioneer', 'pylint', 'pytest', 'mock', 'pytest-cov', + 'coverage', 'numpy', 'pandas', 'matplotlib'], }, # To provide executable scripts, use entry points in preference to the @@ -79,6 +80,7 @@ ], 'pyls': [ 'autopep8 = pyls.plugins.autopep8_format', + 'folding = pyls.plugins.folding', 'flake8 = pyls.plugins.flake8_lint', 'importmagic = pyls.plugins.importmagic_lint', 'jedi_completion = pyls.plugins.jedi_completion', @@ -96,7 +98,7 @@ 'pylint = pyls.plugins.pylint_lint', 'rope_completion = pyls.plugins.rope_completion', 'rope_rename = pyls.plugins.rope_rename', - 'yapf = pyls.plugins.yapf_format', + 'yapf = pyls.plugins.yapf_format' ] }, ) diff --git a/test/plugins/test_completion.py b/test/plugins/test_completion.py index c2feba48..78df9bfa 100644 --- a/test/plugins/test_completion.py +++ b/test/plugins/test_completion.py @@ -45,8 +45,6 @@ def test_rope_import_completion(config, workspace): assert items is None -@pytest.mark.skipif(LooseVersion(jedi.__version__) < LooseVersion('0.14.0'), - reason='This test fails with previous versions of jedi') def test_jedi_completion(config): # Over 'i' in os.path.isabs(...) com_position = {'line': 1, 'character': 15} @@ -118,3 +116,37 @@ def test_jedi_method_completion(config): assert 'insertTextFormat' not in everyone_method assert everyone_method['insertText'] == 'everyone' + + +@pytest.mark.skipif(LooseVersion('0.15.0') <= LooseVersion(jedi.__version__) < LooseVersion('0.16.0'), + reason='This test fails with Jedi 0.15') +def test_numpy_completions(config): + doc_numpy = "import numpy as np; np." + com_position = {'line': 0, 'character': len(doc_numpy)} + doc = Document(DOC_URI, doc_numpy) + items = pyls_jedi_completions(config, doc, com_position) + + assert items + assert any(['array' in i['label'] for i in items]) + + +@pytest.mark.skipif(LooseVersion('0.15.0') <= LooseVersion(jedi.__version__) < LooseVersion('0.16.0'), + reason='This test fails with Jedi 0.15') +def test_pandas_completions(config): + doc_pandas = "import pandas as pd; pd." + com_position = {'line': 0, 'character': len(doc_pandas)} + doc = Document(DOC_URI, doc_pandas) + items = pyls_jedi_completions(config, doc, com_position) + + assert items + assert any(['DataFrame' in i['label'] for i in items]) + + +def test_matplotlib_completions(config): + doc_mpl = "import matplotlib.pyplot as plt; plt." + com_position = {'line': 0, 'character': len(doc_mpl)} + doc = Document(DOC_URI, doc_mpl) + items = pyls_jedi_completions(config, doc, com_position) + + assert items + assert any(['plot' in i['label'] for i in items]) diff --git a/test/plugins/test_definitions.py b/test/plugins/test_definitions.py index f192d0ac..e2db9c6f 100644 --- a/test/plugins/test_definitions.py +++ b/test/plugins/test_definitions.py @@ -1,8 +1,4 @@ # Copyright 2017 Palantir Technologies, Inc. -from distutils.version import LooseVersion -import jedi -import pytest - from pyls import uris from pyls.plugins.definition import pyls_definitions from pyls.workspace import Document @@ -38,15 +34,13 @@ def test_definitions(config): assert [{'uri': DOC_URI, 'range': def_range}] == pyls_definitions(config, doc, cursor_pos) -@pytest.mark.skipif(LooseVersion(jedi.__version__) < LooseVersion('0.14.0'), - reason='This test fails with previous versions of jedi') def test_builtin_definition(config): # Over 'i' in dict cursor_pos = {'line': 8, 'character': 24} # No go-to def for builtins doc = Document(DOC_URI, DOC) - assert len(pyls_definitions(config, doc, cursor_pos)) == 1 + assert not pyls_definitions(config, doc, cursor_pos) def test_assignment(config): diff --git a/test/plugins/test_folding.py b/test/plugins/test_folding.py new file mode 100644 index 00000000..2ee5a9d9 --- /dev/null +++ b/test/plugins/test_folding.py @@ -0,0 +1,164 @@ +# Copyright 2019 Palantir Technologies, Inc. + +from textwrap import dedent + +from pyls import uris +from pyls.workspace import Document +from pyls.plugins.folding import pyls_folding_range + + +DOC_URI = uris.from_fs_path(__file__) +DOC = dedent(""" +def func(arg1, arg2, arg3, + arg4, arg5, default=func( + 2, 3, 4 + )): + return (2, 3, + 4, 5) + +@decorator( + param1, + param2 +) +def decorated_func(x, y, z): + if x: + return y + elif y: + return z + elif x + y > z: + return True + else: + return x + +class A(): + def method(self, x1): + def inner(): + return x1 + + if x2: + func(3, 4, 5, 6, + 7) + elif x3 < 2: + pass + else: + more_complex_func(2, 3, 4, 5, 6, + 8) + return inner + +a = 2 +operation = (a_large_variable_that_fills_all_space + + other_embarrasingly_long_variable - 2 * 3 / 5) + +(a, b, c, + d, e, f) = func(3, 4, 5, 6, + 7, 8, 9, 10) + +for i in range(0, 3): + i += 1 + while x < i: + expr = (2, 4) + a = func(expr + i, arg2, arg3, arg4, + arg5, var(2, 3, 4, + 5)) + for j in range(0, i): + if i % 2 == 1: + pass + +compren = [x for x in range(0, 3) + if x == 2] + +with open('doc', 'r') as f: + try: + f / 0 + except: + pass + finally: + raise SomeException() +""") + +SYNTAX_ERR = dedent(""" +def func(arg1, arg2, arg3, + arg4, arg5, default=func( + 2, 3, 4 + )): + return (2, 3, + 4, 5) + +class A(: + pass + +a = 2 +operation = (a_large_variable_that_fills_all_space + + other_embarrasingly_long_variable - 2 * 3 / + +(a, b, c, + d, e, f) = func(3, 4, 5, 6, + 7, 8, 9, 10 +a = 2 +for i in range(0, 3) + i += 1 + while x < i: + expr = (2, 4) + a = func(expr + i, arg2, arg3, arg4, + arg5, var(2, 3, 4, + 5)) + for j in range(0, i): + if i % 2 == 1: + pass +""") + + +def test_folding(): + doc = Document(DOC_URI, DOC) + ranges = pyls_folding_range(doc) + expected = [{'startLine': 1, 'endLine': 6}, + {'startLine': 2, 'endLine': 3}, + {'startLine': 5, 'endLine': 6}, + {'startLine': 8, 'endLine': 11}, + {'startLine': 12, 'endLine': 20}, + {'startLine': 13, 'endLine': 14}, + {'startLine': 15, 'endLine': 16}, + {'startLine': 17, 'endLine': 18}, + {'startLine': 19, 'endLine': 20}, + {'startLine': 22, 'endLine': 35}, + {'startLine': 23, 'endLine': 35}, + {'startLine': 24, 'endLine': 25}, + {'startLine': 27, 'endLine': 29}, + {'startLine': 28, 'endLine': 29}, + {'startLine': 30, 'endLine': 31}, + {'startLine': 32, 'endLine': 34}, + {'startLine': 33, 'endLine': 34}, + {'startLine': 38, 'endLine': 39}, + {'startLine': 41, 'endLine': 43}, + {'startLine': 42, 'endLine': 43}, + {'startLine': 45, 'endLine': 54}, + {'startLine': 47, 'endLine': 51}, + {'startLine': 49, 'endLine': 51}, + {'startLine': 50, 'endLine': 51}, + {'startLine': 52, 'endLine': 54}, + {'startLine': 53, 'endLine': 54}, + {'startLine': 56, 'endLine': 57}, + {'startLine': 59, 'endLine': 65}, + {'startLine': 60, 'endLine': 61}, + {'startLine': 62, 'endLine': 63}, + {'startLine': 64, 'endLine': 65}] + assert ranges == expected + + +def test_folding_syntax_error(): + doc = Document(DOC_URI, SYNTAX_ERR) + ranges = pyls_folding_range(doc) + expected = [{'startLine': 1, 'endLine': 6}, + {'startLine': 2, 'endLine': 3}, + {'startLine': 5, 'endLine': 6}, + {'startLine': 8, 'endLine': 9}, + {'startLine': 12, 'endLine': 13}, + {'startLine': 15, 'endLine': 17}, + {'startLine': 16, 'endLine': 17}, + {'startLine': 19, 'endLine': 28}, + {'startLine': 21, 'endLine': 25}, + {'startLine': 23, 'endLine': 25}, + {'startLine': 24, 'endLine': 25}, + {'startLine': 26, 'endLine': 28}, + {'startLine': 27, 'endLine': 28}] + assert ranges == expected diff --git a/test/plugins/test_hover.py b/test/plugins/test_hover.py index 9b56d2e2..f34c3513 100644 --- a/test/plugins/test_hover.py +++ b/test/plugins/test_hover.py @@ -1,5 +1,7 @@ # Copyright 2017 Palantir Technologies, Inc. -from pyls import uris +from distutils.version import LooseVersion + +from pyls import uris, _utils from pyls.plugins.hover import pyls_hover from pyls.workspace import Document @@ -20,8 +22,13 @@ def test_hover(): doc = Document(DOC_URI, DOC) + if LooseVersion(_utils.JEDI_VERSION) >= LooseVersion('0.15.0'): + contents = [{'language': 'python', 'value': 'main()'}, 'hello world'] + else: + contents = 'main()\n\nhello world' + assert { - 'contents': [{'language': 'python', 'value': 'main()'}, 'hello world'] + 'contents': contents } == pyls_hover(doc, hov_position) assert {'contents': ''} == pyls_hover(doc, no_hov_position) diff --git a/test/plugins/test_symbols.py b/test/plugins/test_symbols.py index 5377262e..2b6d0c7c 100644 --- a/test/plugins/test_symbols.py +++ b/test/plugins/test_symbols.py @@ -1,8 +1,4 @@ # Copyright 2017 Palantir Technologies, Inc. -from distutils.version import LooseVersion -import jedi -import pytest - from pyls import uris from pyls.plugins.symbols import pyls_document_symbols from pyls.lsp import SymbolKind @@ -25,8 +21,6 @@ def main(x): """ -@pytest.mark.skipif(LooseVersion(jedi.__version__) < LooseVersion('0.14.0'), - reason='This test fails with previous versions of jedi') def test_symbols(config): doc = Document(DOC_URI, DOC) config.update({'plugins': {'jedi_symbols': {'all_scopes': False}}})