feat: implement @() - python-expr operator

peg_parser/parser/subheader.py

@@ -656,6 +656,9 @@ def proc_args(self, args: list[TokenInfo | ast.AST]) -> Iterator[ast.AST]:
         stash: list[TokenInfo] = []
         for ar in args:
             if isinstance(ar, ast.AST):
+                if stash:
+                    yield Parser.toks_to_constant(stash)
+                    stash.clear()
                 yield ar
             elif isinstance(ar, TokenInfo):  # tokens
                 if stash and not ar.is_next_to(stash[-1]):
@@ -682,6 +685,13 @@ def proc_inject(self, args: ast.List, **locs) -> ast.Starred:
             **locs,
         )
 
+    def proc_pyexpr(self, args: ast.List, **locs) -> ast.Starred:
+        return ast.Starred(
+            value=xonsh_call("__xonsh__.list_of_strs_or_callables", args, **locs),
+            ctx=Load,
+            **locs,
+        )
+
     def _build_syntax_error(
         self,
         message: str,

peg_parser/parser/xonsh.gram

@@ -949,19 +949,25 @@ primary:
         )
      }
     | a=primary '[' b=slices ']' { ast.Subscript(value=a, slice=b, ctx=Load, LOCATIONS) }
+    | sub_procs
+    | atom
+    | env_atom
+
+sub_procs:
     # $(...)
-    | a=DOLLAR_LPAREN args=proc_cmds ')' { self.subproc(a, args, LOCATIONS) }
+    | a=DOLLAR_LPAREN ~ args=proc_cmds ')' { self.subproc(a, args, LOCATIONS) }
     # $[...]
-    | a=DOLLAR_LBRACKET args=proc_cmds ']' { self.subproc(a, args, LOCATIONS) }
+    | a=DOLLAR_LBRACKET ~ args=proc_cmds ']' { self.subproc(a, args, LOCATIONS) }
     # ![...]
-    | a=BANG_LBRACKET args=proc_cmds ']' { self.subproc(a, args, LOCATIONS) }
+    | a=BANG_LBRACKET ~ args=proc_cmds ']' { self.subproc(a, args, LOCATIONS) }
     # !(...)
-    | a=BANG_LPAREN args=proc_cmds ')' { self.subproc(a, args, LOCATIONS) }
+    | a=BANG_LPAREN ~ args=proc_cmds ')' { self.subproc(a, args, LOCATIONS) }
+
+env_atom:
     # $a
     | DOLLAR a=NAME { self.expand_env_name(a, LOCATIONS) }
     # ${...}
     | DOLLAR_LBRACE a=slices '}' { self.expand_env_expr(a, LOCATIONS) }
-    | atom
 
 proc_cmds:
     | a=proc_cmd+ { ast.List(elts=list(self.proc_args(a)), ctx=Load, LOCATIONS) }
@@ -970,7 +976,10 @@ proc_cmd:
     | NAME
     | OP
     | STRING
-    | AT_DOLLAR_LPAREN a=proc_cmds ')' { self.proc_inject(a, LOCATIONS) }
+    | AT_LPAREN ~ a=(bare_genexp | expressions) ')' { self.proc_pyexpr(a, LOCATIONS) }
+    | AT_DOLLAR_LPAREN ~ a=proc_cmds ')' { self.proc_inject(a, LOCATIONS) }
+    | env_atom
+    | sub_procs
 
 slices:
     | a=slice !',' { a }
@@ -1168,11 +1177,14 @@ setcomp[ast.SetComp]:
     | invalid_comprehension
 
 genexp[ast.GeneratorExp]:
-    | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' {
+    | '(' bare_genexp ')' {
         ast.GeneratorExp(elt=a, generators=b, LOCATIONS)
      }
     | invalid_comprehension
 
+bare_genexp:
+    | a=( assignment_expression | expression !':=') b=for_if_clauses { ast.GeneratorExp(elt=a, generators=b, LOCATIONS) }
+
 dictcomp[ast.DictComp]:
     | '{' a=kvpair b=for_if_clauses '}' { ast.DictComp(key=a[0], value=a[1], generators=b, LOCATIONS) }
     | invalid_dict_comprehension

tests/conftest.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import ast
 import contextlib
 import io
 import logging
@@ -13,9 +14,6 @@
 
 
 def nodes_equal(x, y):
-    import ast
-
-    __tracebackhide__ = True
     assert type(x) == type(y), f"Ast nodes do not have the same type: '{type(x)}' != '{type(y)}' "
     if isinstance(x, ast.Constant):
         assert x.value == y.value, (
@@ -87,8 +85,6 @@ def factory(text, verbose=False, mode="eval", py_version: tuple | None = None):
 
 @pytest.fixture
 def check_ast(parse_str):
-    import ast
-
     def factory(inp: str, mode="eval", verbose=False):
         # expect a Python AST
         exp = ast.parse(inp, mode=mode)
@@ -113,8 +109,6 @@ def factory(text: str, mode="eval", **locs):
 @pytest.fixture
 def unparse_diff(parse_str):
     def factory(text: str, right: str | None = None, mode="eval"):
-        import ast
-
         left = parse_str(text, mode=mode)
         left = ast.unparse(left)
         if right is None:
@@ -127,7 +121,28 @@ def factory(text: str, right: str | None = None, mode="eval"):
 
 @pytest.fixture
 def xsh():
-    return MagicMock()
+    obj = MagicMock()
+
+    def list_of_strs_or_callables(x):
+        """
+        A simplified version of the xonsh function.
+        """
+        if isinstance(x, (str, bytes)):
+            return [x]
+        if callable(x):
+            return [x([])]
+        return x
+
+    def subproc_captured(cmds):
+        return "-".join([str(item) for item in cmds])
+
+    def subproc_captured_inject(cmds):
+        return cmds
+
+    obj.list_of_strs_or_callables = MagicMock(wraps=list_of_strs_or_callables)
+    obj.subproc_captured = MagicMock(wraps=subproc_captured)
+    obj.subproc_captured_inject = MagicMock(wraps=subproc_captured_inject)
+    return obj
 
 
 @pytest.fixture

tests/data/exprs/proc_injection.py

@@ -2,4 +2,4 @@
 __xonsh__.subproc_uncaptured([*__xonsh__.subproc_captured_inject(['which', 'python'])])
 
 # $[ls @$(dirname @$(which python))]
-__xonsh__.subproc_uncaptured([*__xonsh__.subproc_captured_inject([*__xonsh__.subproc_captured_inject(['which', 'python']), 'dirname']), 'ls'])
+__xonsh__.subproc_uncaptured(['ls', *__xonsh__.subproc_captured_inject(['dirname', *__xonsh__.subproc_captured_inject(['which', 'python'])])])

tests/test_parser.py

@@ -235,7 +235,7 @@ def test_stmts(file, unparse_diff, subtests):
 def test_statements(check_xonsh_ast, inp):
     if not inp.endswith("\n"):
         inp += "\n"
-    return check_xonsh_ast(inp, mode="exec")
+    check_xonsh_ast(inp, mode="exec")
 
 
 @pytest.mark.parametrize(
@@ -246,22 +246,30 @@ def test_statements(check_xonsh_ast, inp):
         ("$( ls )", ["ls"]),
         ("$( ls)", ["ls"]),
         ("$(ls .)", ["ls", "."]),
-        ('$(ls ".")', []),
+        ('$(ls ".")', ["ls", '"."']),
         ("$(ls -l)", ["ls", "-l"]),
-        ("$(ls $WAKKA)", []),
-        ('$(ls @(None or "."))', []),
-        ('$(echo hello | @(lambda a, s=None: "hey!") foo bar baz)', ""),
-        ("$(echo @(i**2 for i in range(20)))", ""),
-        ("$(echo @('a', 7))", ""),
-        ("$(@$(which echo) ls " "| @(lambda a, s=None: $(@(s.strip()) @(a[1]))) foo -la baz)", ""),
-        ("![echo /x/@(y)/z]", []),
-        ("$(ls $(ls))", []),
-        ("$(ls $(ls) -l)", []),
+        ("$(ls $WAKKA)", ["ls", "wak"]),
+        ('$(ls @(None or "."))', ["ls", "."]),
+        (
+            '$(echo hello | @(lambda a, s=None: "hey!") foo bar baz)',
+            ["echo", "hello", "|", "hey!", "foo", "bar", "baz"],
+        ),
+        (
+            "$(echo @(i**2 for i in range(20) ) )",
+            ["echo", 0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100, 121, 144, 169, 196, 225, 256, 289, 324, 361],
+        ),
+        ("$(echo @('a', 7))", ["echo", "a", 7]),
+        pytest.param(
+            "$(@$(which echo) ls | @(lambda a, s=None: $(@(s.strip()) @(a[1]))) foo -la baz)",
+            "",
+            marks=pytest.mark.xfail,
+        ),
+        ("$(ls $(ls))", ["ls", "ls"]),
+        ("$(ls $(ls) -l)", ["ls", "ls", "-l"]),
     ],
 )
-@pytest.mark.xfail
 def test_captured_procs(inp, args, check_xonsh_ast, xsh):
-    check_xonsh_ast(inp)
+    check_xonsh_ast(inp, mode="exec", xenv={"WAKKA": "wak"})
     xsh.subproc_captured.assert_called_with(args)