Fixes and 0.20.4

grammar.js

@@ -55,6 +55,7 @@ module.exports = grammar({
     [$.with_item, $._collection_elements],
     [$.named_expression, $.as_pattern],
     [$.print_statement, $.primary_expression],
+    [$.type_alias_statement, $.primary_expression],
   ],
 
   supertypes: $ => [
@@ -132,6 +133,7 @@ module.exports = grammar({
       $.global_statement,
       $.nonlocal_statement,
       $.exec_statement,
+      $.type_alias_statement,
     ),
 
     import_statement: $ => seq(
@@ -412,6 +414,7 @@ module.exports = grammar({
       optional('async'),
       'def',
       field('name', $.identifier),
+      field('type_parameters', optional($.type_parameter)),
       field('parameters', $.parameters),
       optional(
         seq(
@@ -462,13 +465,26 @@ module.exports = grammar({
       ),
     ),
 
+    type_alias_statement: $ => prec.dynamic(1, seq(
+      'type',
+      $.type,
+      '=',
+      $.type,
+    )),
+
     class_definition: $ => seq(
       'class',
       field('name', $.identifier),
+      field('type_parameters', optional($.type_parameter)),
       field('superclasses', optional($.argument_list)),
       ':',
       field('body', $._suite),
     ),
+    type_parameter: $ => seq(
+      '[',
+      commaSep1($.type),
+      ']',
+    ),
 
     parenthesized_list_splat: $ => prec(PREC.parenthesized_list_splat, seq(
       '(',
@@ -842,7 +858,19 @@ module.exports = grammar({
       field('type', $.type),
     )),
 
-    type: $ => $.expression,
+    type: $ => choice(
+      $.expression,
+      $.splat_type,
+      $.generic_type,
+      $.union_type,
+      $.constrained_type,
+      $.member_type,
+    ),
+    splat_type: $ => prec(1, seq(choice('*', '**'), $.identifier)),
+    generic_type: $ => prec(1, seq($.identifier, $.type_parameter)),
+    union_type: $ => prec.left(seq($.type, '|', $.type)),
+    constrained_type: $ => prec.right(seq($.type, ':', $.type)),
+    member_type: $ => seq($.type, '.', $.identifier),
 
     keyword_argument: $ => seq(
       field('name', choice($.identifier, $.keyword_identifier)),
@@ -1055,16 +1083,19 @@ module.exports = grammar({
 
     identifier: _ => /[_\p{XID_Start}][_\p{XID_Continue}]*/,
 
-    keyword_identifier: $ => prec(-3, alias(
-      choice(
-        'print',
-        'exec',
-        'async',
-        'await',
-        'match',
-      ),
-      $.identifier,
-    )),
+    keyword_identifier: $ => choice(
+      prec(-3, alias(
+        choice(
+          'print',
+          'exec',
+          'async',
+          'await',
+          'match',
+        ),
+        $.identifier,
+      )),
+      alias('type', $.identifier),
+    ),
 
     true: _ => 'True',
     false: _ => 'False',

script/known_failures.txt

@@ -0,0 +1,4 @@
+examples/cpython/Lib/test/badsyntax_3131.py
+examples/cpython/Lib/test/badsyntax_future8.py
+examples/cpython/Lib/test/test_compile.py
+examples/cpython/Tools/build/generate_re_casefix.py

script/parse-examples

@@ -1,36 +1,47 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-set -e
+set -eu
 
 cd "$(dirname "$0")/.."
 
-function checkout() {
-  repo=$1; url=$2; sha=$3
-
-  if [ ! -d "$repo" ]; then
-    git clone "https://github.com/$url" "$repo"
-  fi
-
-  pushd "$repo"
-  git fetch && git reset --hard "$sha"
-  popd
+function clone_repo {
+	owner=$1
+	name=$2
+	sha=$3
+
+	path=examples/$name
+	if [ ! -d "$path" ]; then
+		echo "Cloning $owner/$name"
+		git clone "https://github.com/$owner/$name" "$path"
+	fi
+
+	pushd "$path" >/dev/null
+	actual_sha=$(git rev-parse HEAD)
+	if [ "$actual_sha" != "$sha" ]; then
+		echo "Updating $owner/$name to $sha"
+		git fetch
+		git reset --hard "$sha"
+	fi
+	popd >/dev/null
 }
 
-checkout examples/numpy numpy/numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
-checkout examples/django django/django 01974d7f7549b2dca2a729c3c1a1ea7d4585eb3a
-checkout examples/flask pallets/flask de464c03e134127140e5622e230790806a133ff9
+clone_repo numpy numpy 058851c5cfc98f50f11237b1c13d77cfd1f40475
+clone_repo django django 01974d7f7549b2dca2a729c3c1a1ea7d4585eb3a
+clone_repo pallets flask de464c03e134127140e5622e230790806a133ff9
+clone_repo python cpython bb456a08a3db851e6feaefc3328f39096919ec8d
 
 known_failures="$(cat script/known_failures.txt)"
 
+# shellcheck disable=2046
 tree-sitter parse -q \
-  'examples/**/*.py' \
-  $(for file in $known_failures; do echo "!${file}"; done)
+	'examples/**/*.py' \
+	$(for file in $known_failures; do echo "!${file}"; done)
 
 example_count=$(find examples -name '*.py' | wc -l)
-failure_count=$(wc -w <<< "$known_failures")
-success_count=$(( $example_count - $failure_count ))
-success_percent=$(bc -l <<< "100*${success_count}/${example_count}")
+failure_count=$(wc -w <<<"$known_failures")
+success_count=$((example_count - failure_count))
+success_percent=$(bc -l <<<"100*${success_count}/${example_count}")
 
 printf \
-  "Successfully parsed %d of %d example files (%.1f%%)\n" \
-  $success_count $example_count $success_percent
+	"Successfully parsed %d of %d example files (%.1f%%)\n" \
+	"$success_count" "$example_count" "$success_percent"