Type hints sometimes break type inference

[gpelouze]

Extracting the following code:

def rewrite_list_nested(in_list: List[Object],concurrency: int): -> List[List[Object]]
    out_list = []
    in_list_len = len(in_list)
    worker_chunk_size = math.floor(in_list_len / concurrency)
    leftovers = in_list_len%param_concurrency
    while in_list:
        leftover = 0
        if leftovers:
            leftover = 1
            leftovers -= leftover
        out_list.append(in_list[:worker_chunk_size+leftover])
        in_list = in_list[worker_chunk_size+leftover:]
    return out_list

results in an error:

[E 2024-01-19 12:21:25.212 ServerApp] Uncaught exception POST /vl-vol2bird/user/[user]/vre/containerizer/extract?[id] (10.244.7.0)
    HTTPServerRequest(protocol='https', host='naavre.lifewatch.dev', method='POST', uri='/vl-vol2bird/user/[user]/vre/containerizer/extract?[id]', version='HTTP/1.1', remote_ip='10.244.7.0')
    Traceback (most recent call last):
      File "/venv/lib/python3.11/site-packages/pytype/io.py", line 336, in wrap_pytype_exceptions
        yield
      File "/venv/lib/python3.11/site-packages/pytype/tools/annotate_ast/annotate_ast.py", line 40, in infer_types
        return traces.trace(source, options)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/tools/traces/traces.py", line 85, in trace
        ret = analyze.infer_types(
              ^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/analyze.py", line 103, in infer_types
        loc, defs = ctx.vm.run_program(src, filename, init_maximum_depth)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/vm.py", line 557, in run_program
        src_tree = directors.parse_src(src, self.ctx.python_version)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/directors/parser.py", line 462, in parse_src
        ast.parse(src, feature_version=python_version[1]), _process_comments(src))  # pylint: disable=unexpected-keyword-arg
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/ast.py", line 50, in parse
        return compile(source, filename, mode, flags,
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "<unknown>", line 12
        def rewrite_list_nested(in_list: List[Object],concurrency: int): -> List[List[Object]]
                                                                         ^^
    SyntaxError: invalid syntax

    The above exception was the direct cause of the following exception:

    Traceback (most recent call last):
      File "/venv/lib/python3.11/site-packages/tornado/web.py", line 1786, in _execute
        result = await result
                 ^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/jupyterlab_vre/component_containerizer/handlers.py", line 78, in post
        extractor = PyExtractor(notebook)
                    ^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/jupyterlab_vre/services/extractor/pyextractor.py", line 22, in __init__
        self.notebook_names = self.__extract_cell_names(
                              ^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/jupyterlab_vre/services/extractor/pyextractor.py", line 183, in __extract_cell_names
        tree = self.__get_annotated_ast(cell_source)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/jupyterlab_vre/services/extractor/pyextractor.py", line 141, in __get_annotated_ast
        return annotate_ast.annotate_source(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/tools/annotate_ast/annotate_ast.py", line 20, in annotate_source
        source_code = infer_types(source, pytype_options)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/venv/lib/python3.11/site-packages/pytype/tools/annotate_ast/annotate_ast.py", line 39, in infer_types
        with io.wrap_pytype_exceptions(PytypeError, filename=options.input):
      File "/venv/lib/python3.11/contextlib.py", line 158, in __exit__
        self.gen.throw(typ, value, traceback)
      File "/venv/lib/python3.11/site-packages/pytype/io.py", line 348, in wrap_pytype_exceptions
        raise exception_type("Error reading file %s at line %s: %s" %
    pytype.tools.annotate_ast.annotate_ast.PytypeError: Error reading file dummpy_input_file at line 12: invalid syntax
[W 2024-01-19 12:21:25.213 ServerApp] Unhandled error

Removing the type hints, the cell can be extracted without issue:

def rewrite_list_nested(in_list,concurrency):
    out_list = []
    in_list_len = len(in_list)
    worker_chunk_size = math.floor(in_list_len / concurrency)
    leftovers = in_list_len%param_concurrency
    while in_list:
        leftover = 0
        if leftovers:
            leftover = 1
            leftovers -= leftover
        out_list.append(in_list[:worker_chunk_size+leftover])
        in_list = in_list[worker_chunk_size+leftover:]
    return out_list