python_units.gram

# PEG grammar for Python

@class PythonParser

@subheader'''
import enum
import io
import itertools
import os
import sys
import token
from typing import (
    Any, Callable, Iterator, List, Literal, NoReturn, Sequence, Tuple, TypeVar, Union
)

from pegen.tokenizer import Tokenizer
from unit_syntax._parser_support import (utf8_locations, character_locations, formatted_field,
                                         interpolated_string, concatenate_templates, type_parameter)

# Singleton ast nodes, created once for efficiency
Load = ast.Load()
Store = ast.Store()
Del = ast.Del()

Node = TypeVar("Node")
FC = TypeVar("FC", ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)

EXPR_NAME_MAPPING = {
    ast.Attribute: "attribute",
    ast.Subscript: "subscript",
    ast.Starred: "starred",
    ast.Name: "name",
    ast.List: "list",
    ast.Tuple: "tuple",
    ast.Lambda: "lambda",
    ast.Call: "function call",
    ast.BoolOp: "expression",
    ast.BinOp: "expression",
    ast.UnaryOp: "expression",
    ast.GeneratorExp: "generator expression",
    ast.Yield: "yield expression",
    ast.YieldFrom: "yield expression",
    ast.Await: "await expression",
    ast.ListComp: "list comprehension",
    ast.SetComp: "set comprehension",
    ast.DictComp: "dict comprehension",
    ast.Dict: "dict literal",
    ast.Set: "set display",
    ast.JoinedStr: "f-string expression",
    ast.FormattedValue: "f-string expression",
    ast.Compare: "comparison",
    ast.IfExp: "conditional expression",
    ast.NamedExpr: "named expression",
}


def parse_file(
    path: str,
    py_version: Optional[tuple]=None,
    token_stream_factory: Optional[
        Callable[[Callable[[], str]], Iterator[tokenize.TokenInfo]]
    ] = None,
    verbose:bool = False,
) -> ast.Module:
    """Parse a file."""
    with open(path, encoding="utf-8") as f:
        source = f.read()
        f.seek(0)
        tok_stream = (
            token_stream_factory(f.readline)
            if token_stream_factory else
            tokenize.generate_tokens(f.readline)
        )
        tokenizer = Tokenizer(tok_stream, verbose=verbose, path=path)
        parser = PythonParser(
            tokenizer,
            verbose=verbose,
            filename=os.path.basename(path),
            py_version=py_version
        )
        parser.source = source
        return utf8_locations(parser.parse("file"), source)


def parse_string(
    source: str,
    mode: Union[Literal["eval"], Literal["exec"]],
    py_version: Optional[tuple]=None,
    token_stream_factory: Optional[
        Callable[[Callable[[], str]], Iterator[tokenize.TokenInfo]]
    ] = None,
    verbose:bool = False,
) -> Any:
    """Parse a string."""
    tok_stream = (
        token_stream_factory(io.StringIO(source).readline)
        if token_stream_factory else
        tokenize.generate_tokens(io.StringIO(source).readline)
    )
    tokenizer = Tokenizer(tok_stream, verbose=verbose)
    parser = PythonParser(tokenizer, verbose=verbose, py_version=py_version)
    parser.source = source
    return utf8_locations(parser.parse(mode if mode == "eval" else "file"), source)


class Target(enum.Enum):
    FOR_TARGETS = enum.auto()
    STAR_TARGETS = enum.auto()
    DEL_TARGETS = enum.auto()


class Parser(Parser):

    #: Name of the source file, used in error reports
    filename : str

    def __init__(self,
        tokenizer: Tokenizer, *,
        verbose: bool = False,
        filename: str = "<unknown>",
        py_version: Optional[tuple] = None,
    ) -> None:
        super().__init__(tokenizer, verbose=verbose)
        self.filename = filename
        self.py_version = min(py_version, sys.version_info) if py_version else sys.version_info

    def parse(self, rule: str, call_invalid_rules: bool = False) -> Optional[ast.AST]:
        old = self.call_invalid_rules
        self.call_invalid_rules = call_invalid_rules
        res = getattr(self, rule)()

        if res is None:

            # Grab the last token that was parsed in the first run to avoid
            # polluting a generic error reports with progress made by invalid rules.
            last_token = self._tokenizer.diagnose()

            if not call_invalid_rules:
                self.call_invalid_rules = True

                # Reset the parser cache to be able to restart parsing from the
                # beginning.
                self._reset(0)  # type: ignore
                self._cache.clear()

                res = getattr(self, rule)()

            self.raise_raw_syntax_error("invalid syntax", last_token.start, last_token.end)

        return res

    def check_version(self, min_version: Tuple[int, ...], error_msg: str, node: Node) -> Node:
        """Check that the python version is high enough for a rule to apply.

        """
        if self.py_version >= min_version:
            return node
        else:
            raise SyntaxError(
                f"{error_msg} is only supported in Python {min_version} and above."
            )

    def raise_indentation_error(self, msg: str) -> None:
        """Raise an indentation error."""
        last_token = self._tokenizer.diagnose()
        args = (self.filename, last_token.start[0], last_token.start[1] + 1, last_token.line)
        if sys.version_info >= (3, 10):
            args += (last_token.end[0], last_token.end[1] + 1)
        raise IndentationError(msg, args)

    def get_expr_name(self, node) -> str:
        """Get a descriptive name for an expression."""
        # See https://github.com/python/cpython/blob/master/Parser/pegen.c#L161
        assert node is not None
        node_t = type(node)
        if node_t is ast.Constant:
            v = node.value
            if v is Ellipsis:
                return "ellipsis"
            elif v is None:
                return str(v)
            # Avoid treating 1 as True through == comparison
            elif v is True:
                return str(v)
            elif v is False:
                return str(v)
            else:
                return "literal"

        try:
            return EXPR_NAME_MAPPING[node_t]
        except KeyError:
            raise ValueError(
                f"unexpected expression in assignment {type(node).__name__} "
                f"(line {node.lineno})."
            )

    def get_invalid_target(self, target: Target, node: Optional[ast.AST]) -> Optional[ast.AST]:
        """Get the meaningful invalid target for different assignment type."""
        if node is None:
            return None

        # We only need to visit List and Tuple nodes recursively as those
        # are the only ones that can contain valid names in targets when
        # they are parsed as expressions. Any other kind of expression
        # that is a container (like Sets or Dicts) is directly invalid and
        # we do not need to visit it recursively.
        if isinstance(node, (ast.List, ast.Tuple)):
            for e in node.elts:
                if (inv := self.get_invalid_target(target, e)) is not None:
                    return inv
        elif isinstance(node, ast.Starred):
            if target is Target.DEL_TARGETS:
                return node
            return self.get_invalid_target(target, node.value)
        elif isinstance(node, ast.Compare):
            # This is needed, because the `a in b` in `for a in b` gets parsed
            # as a comparison, and so we need to search the left side of the comparison
            # for invalid targets.
            if target is Target.FOR_TARGETS:
                if isinstance(node.ops[0], ast.In):
                    return self.get_invalid_target(target, node.left)
                return None

            return node
        elif isinstance(node, (ast.Name, ast.Subscript, ast.Attribute)):
            return None
        else:
            return node

    def set_expr_context(self, node, context):
        """Set the context (Load, Store, Del) of an ast node."""
        node.ctx = context
        return node

    def ensure_real(self, number: tokenize.TokenInfo) -> float:
        value = ast.literal_eval(number.string)
        if type(value) is complex:
            self.raise_syntax_error_known_location("real number required in complex literal", number)
        return value

    def ensure_imaginary(self, number: tokenize.TokenInfo) -> complex:
        value = ast.literal_eval(number.string)
        if type(value) is not complex:
            self.raise_syntax_error_known_location("imaginary number required in complex literal", number)
        return value

    def check_fstring_conversion(self, mark: tokenize.TokenInfo, name: tokenize.TokenInfo) -> int:
        if mark.end != name.start:
            self.raise_syntax_error_known_range(
                "f-string: conversion type must come right after the exclamanation mark",
                mark,
                name
            )

        s = name.string
        if len(s) > 1 or s not in ("s", "r", "a"):
            self.raise_syntax_error_known_location(
                f"f-string: invalid conversion character '{s}': expected 's', 'r', or 'a'",
                name,
            )

        return ord(s)

    def _concat_strings_in_constant(self, parts) -> Union[str, bytes]:
        s = ast.literal_eval(parts[0].string)
        for ss in parts[1:]:
            s += ast.literal_eval(ss.string)
        args = dict(
            value=s,
            lineno=parts[0].start[0],
            col_offset=parts[0].start[1],
            end_lineno=parts[-1].end[0],
            end_col_offset=parts[0].end[1],
        )
        if parts[0].string.startswith("u"):
            args["kind"] = "u"
        return ast.Constant(**args)


    def concatenate_strings(self, parts):
        """Concatenate multiple tokens and ast.JoinedStr"""
        # Get proper start and stop
        start = end = None
        if isinstance(parts[0], ast.JoinedStr):
            start = parts[0].lineno, parts[0].col_offset
        if isinstance(parts[-1], ast.JoinedStr):
            end = parts[-1].end_lineno, parts[-1].end_col_offset

        # Combine the different parts
        seen_joined = False
        values = []
        ss = []
        for p in parts:
            if isinstance(p, ast.JoinedStr):
                seen_joined = True
                if ss:
                    values.append(self._concat_strings_in_constant(ss))
                    ss.clear()
                values.extend(p.values)
            else:
                ss.append(p)

        if ss:
            values.append(self._concat_strings_in_constant(ss))

        consolidated = []
        for p in values:
            if consolidated and isinstance(consolidated[-1], ast.Constant) and isinstance(p, ast.Constant):
                consolidated[-1].value += p.value
                consolidated[-1].end_lineno = p.end_lineno
                consolidated[-1].end_col_offset = p.end_col_offset
            else:
                consolidated.append(p)

        if not seen_joined and len(values) == 1 and isinstance(values[0], ast.Constant):
            return values[0]
        else:
            return ast.JoinedStr(
                values=consolidated,
                lineno=start[0] if start else values[0].lineno,
                col_offset=start[1] if start else values[0].col_offset,
                end_lineno=end[0] if end else values[-1].end_lineno,
                end_col_offset=end[1] if end else values[-1].end_col_offset,
            )

    def generate_ast_for_string(self, tokens):
        """Generate AST nodes for strings."""
        err_args = None
        line_offset = tokens[0].start[0]
        line = line_offset
        col_offset = 0
        source = "(\\n"
        for t in tokens:
            n_line = t.start[0] - line
            if n_line:
                col_offset = 0
            source += """\\n""" * n_line + ' ' * (t.start[1] - col_offset) + t.string
            line, col_offset = t.end
        source += "\\n)"
        try:
            m = ast.parse(source)
        except SyntaxError as err:
            args = (err.filename, err.lineno + line_offset - 2, err.offset, err.text)
            if sys.version_info >= (3, 10):
                args += (err.end_lineno + line_offset - 2, err.end_offset)
            err_args = (err.msg, args)
            # Ensure we do not keep the frame alive longer than necessary
            # by explicitly deleting the error once we got what we needed out
            # of it
            del err

        # Avoid getting a triple nesting in the error report that does not
        # bring anything relevant to the traceback.
        if err_args is not None:
            raise SyntaxError(*err_args)

        # The native parser returns UTF-8 columns. Restore character columns
        # before the final shared conversion applies the original source offsets.
        node = character_locations(m.body[0].value, source)
        # Since we asked Python to parse an alterred source starting at line 2
        # we alter the lineno of the returned AST to recover the right line.
        # If the string start at line 1, tha AST says 2 so we need to decrement by 1
        # hence the -2.
        ast.increment_lineno(node, line_offset - 2)
        return node

    def extract_import_level(self, tokens: List[tokenize.TokenInfo]) -> int:
        """Extract the relative import level from the tokens preceding the module name.

        '.' count for one and '...' for 3.

        """
        level = 0
        for t in tokens:
            if t.string == ".":
                level += 1
            else:
                level += 3
        return level

    def set_decorators(self,
        target: FC,
        decorators: list
    ) -> FC:
        """Set the decorators on a function or class definition."""
        target.decorator_list = decorators
        return target

    def get_comparison_ops(self, pairs):
        return [op for op, _ in pairs]

    def get_comparators(self, pairs):
        return [comp for _, comp in pairs]

    def set_arg_type_comment(self, arg, type_comment):
        if type_comment or sys.version_info < (3, 9):
            arg.type_comment = type_comment
        return arg

    def make_arguments(self,
        pos_only: Optional[List[Tuple[ast.arg, None]]],
        pos_only_with_default: List[Tuple[ast.arg, Any]],
        param_no_default: Optional[List[Tuple[ast.arg, None]]],
        param_default: Optional[List[Tuple[ast.arg, Any]]],
        after_star: Optional[Tuple[Optional[ast.arg], List[Tuple[ast.arg, Any]], Optional[ast.arg]]]
    ) -> ast.arguments:
        """Build a function definition arguments."""
        defaults = (
            [d for _, d in pos_only_with_default if d is not None]
            if pos_only_with_default else
            []
        )
        defaults += (
            [d for _, d in param_default if d is not None]
            if param_default else
            []
        )

        pos_only = pos_only or pos_only_with_default

        # Because we need to combine pos only with and without default even
        # the version with no default is a tuple
        pos_only = [p for p, _ in pos_only]
        params = (param_no_default or []) + ([p for p, _ in param_default] if param_default else [])

        # If after_star is None, make a default tuple
        after_star = after_star or (None, [], None)

        return ast.arguments(
            posonlyargs=pos_only,
            args=params,
            defaults=defaults,
            vararg=after_star[0],
            kwonlyargs=[p for p, _ in after_star[1]],
            kw_defaults=[d for _, d in after_star[1]],
            kwarg=after_star[2]
        )

    def _build_syntax_error(
        self,
        message: str,
        start: Optional[Tuple[int, int]] = None,
        end: Optional[Tuple[int, int]] = None
    ) -> SyntaxError:
        line_from_token = start is None and end is None
        if start is None or end is None:
            tok = self._tokenizer.diagnose()
            start = start or tok.start
            end = end or tok.end

        if line_from_token:
            line = tok.line
        else:
            # End is used only to get the proper text
            line = "\\n".join(
                self._tokenizer.get_lines(list(range(start[0], end[0] + 1)))
            )

        # tokenize.py index column offset from 0 while Cpython index column
        # offset at 1 when reporting SyntaxError, so we need to increment
        # the column offset when reporting the error.
        args = (self.filename, start[0], start[1] + 1, line)
        if sys.version_info >= (3, 10):
            args += (end[0], end[1] + 1)

        return SyntaxError(message, args)

    def raise_raw_syntax_error(
        self,
        message: str,
        start: Optional[Tuple[int, int]] = None,
        end: Optional[Tuple[int, int]] = None
    ) -> NoReturn:
        raise self._build_syntax_error(message, start, end)

    def make_syntax_error(self, message: str) -> SyntaxError:
        return self._build_syntax_error(message)

    def expect_forced(self, res: Any, expectation: str) -> Optional[tokenize.TokenInfo]:
        if res is None:
            last_token = self._tokenizer.diagnose()
            end = last_token.start
            if (
                sys.version_info >= (3, 12)
                or (
                    sys.version_info >= (3, 11)
                    and last_token.type != 4
                )
            ):  # i.e. not a \n
                end = last_token.end
            self.raise_raw_syntax_error(
                f"expected {expectation}", last_token.start, end
            )
        return res

    def raise_syntax_error(self, message: str) -> NoReturn:
        """Raise a syntax error."""
        tok = self._tokenizer.diagnose()
        raise self._build_syntax_error(
            message, tok.start,
            tok.end if sys.version_info >= (3, 12) or tok.type != 4 else tok.start
        )

    def raise_syntax_error_known_location(
        self, message: str, node: Union[ast.AST, tokenize.TokenInfo]
    ) -> NoReturn:
        """Raise a syntax error that occured at a given AST node."""
        if isinstance(node, tokenize.TokenInfo):
            start = node.start
            end = node.end
        else:
            start = node.lineno, node.col_offset
            end = node.end_lineno, node.end_col_offset

        raise self._build_syntax_error(message, start, end)

    def raise_syntax_error_known_range(
        self,
        message: str,
        start_node: Union[ast.AST, tokenize.TokenInfo],
        end_node: Union[ast.AST, tokenize.TokenInfo]
    ) -> NoReturn:
        if isinstance(start_node, tokenize.TokenInfo):
            start = start_node.start
        else:
            start = start_node.lineno, start_node.col_offset

        if isinstance(end_node, tokenize.TokenInfo):
            end = end_node.end
        else:
            end = end_node.end_lineno, end_node.end_col_offset

        raise self._build_syntax_error(message, start, end)

    def raise_syntax_error_starting_from(
        self,
        message: str,
        start_node: Union[ast.AST, tokenize.TokenInfo]
    ) -> NoReturn:
        if isinstance(start_node, tokenize.TokenInfo):
            start = start_node.start
        else:
            start = start_node.lineno, start_node.col_offset

        last_token = self._tokenizer.diagnose()

        raise self._build_syntax_error(message, start, last_token.start)

    def raise_syntax_error_invalid_target(
        self, target: Target, node: Optional[ast.AST]
    ) -> None:
        invalid_target = self.get_invalid_target(target, node)

        if invalid_target is None:
            return None

        if target in (Target.STAR_TARGETS, Target.FOR_TARGETS):
            msg = f"cannot assign to {self.get_expr_name(invalid_target)}"
        else:
            msg = f"cannot delete {self.get_expr_name(invalid_target)}"

        self.raise_syntax_error_known_location(msg, invalid_target)

    def raise_syntax_error_on_next_token(self, message: str) -> NoReturn:
        next_token = self._tokenizer.peek()
        raise self._build_syntax_error(message, next_token.start, next_token.end)

class UnitsExpr(ast.expr):
    _fields = ("value", "units")
    _attributes = ("lineno", "col_offset", "end_lineno", "end_col_offset")

'''


# STARTING RULES
# ==============

start: file

file[ast.Module]: a=[statements] ENDMARKER { ast.Module(body=a or [], type_ignores=[]) }
interactive[ast.Interactive]: a=statement_newline { ast.Interactive(body=a) }
eval[ast.Expression]: a=expressions NEWLINE* ENDMARKER { ast.Expression(body=a) }
func_type[ast.FunctionType]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { ast.FunctionType(argtypes=a, returns=b) }
fstring[ast.Expr]: star_expressions

# GENERAL STATEMENTS
# ==================

statements[list]: a=statement+ { list(itertools.chain.from_iterable(a)) }

statement[list]: a=compound_stmt { [a] } | a=simple_stmts { a }

statement_newline[list]:
    | a=compound_stmt NEWLINE { [a] }
    | simple_stmts
    | NEWLINE { [ast.Pass(LOCATIONS)] }
    | ENDMARKER { None }

simple_stmts[list]:
    | a=simple_stmt !';' NEWLINE { [a] } # Not needed, there for speedup
    | a=';'.simple_stmt+ [';'] NEWLINE { a }

# NOTE: assignment MUST precede expression, else parsing a simple assignment
# will throw a SyntaxError.
simple_stmt (memo):
    | &"type" type_alias
    | assignment
    | &('import' | 'from' | "lazy") import_stmt
    | e=star_expressions { ast.Expr(value=e, LOCATIONS) }
    | &'return' return_stmt
    | &'raise' raise_stmt
    | 'pass' { ast.Pass(LOCATIONS) }
    | &'del' del_stmt
    | &'yield' yield_stmt
    | &'assert' assert_stmt
    | 'break' { ast.Break(LOCATIONS) }
    | 'continue' { ast.Continue(LOCATIONS) }
    | &'global' global_stmt
    | &'nonlocal' nonlocal_stmt

compound_stmt:
    | &('def' | '@' | 'async') function_def
    | &'if' if_stmt
    | &('class' | '@') class_def
    | &('with' | 'async') with_stmt
    | &('for' | 'async') for_stmt
    | &'try' try_stmt
    | &'while' while_stmt
    | match_stmt

# SIMPLE STATEMENTS
# =================

# NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield'
assignment:
    | a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] {
        self.check_version(
            (3, 6),
            "Variable annotation syntax is",
            ast.AnnAssign(
                target=ast.Name(
                    id=a.string,
                    ctx=Store,
                    lineno=a.start[0],
                    col_offset=a.start[1],
                    end_lineno=a.end[0],
                    end_col_offset=a.end[1],
                ),
                annotation=b,
                value=c,
                simple=1,
                LOCATIONS,
            )
        ) }
    | a=('(' b=single_target ')' { b }
         | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] {
        self.check_version(
            (3, 6),
            "Variable annotation syntax is",
            ast.AnnAssign(
                target=a,
                annotation=b,
                value=c,
                simple=0,
                LOCATIONS,
            )
        )
     }
    | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] {
         ast.Assign(targets=a, value=b, type_comment=tc, LOCATIONS)
     }
    | a=single_target b=augassign ~ c=(yield_expr | star_expressions) {
        ast.AugAssign(target = a, op=b, value=c, LOCATIONS)
     }
    | invalid_assignment

annotated_rhs: yield_expr | star_expressions

augassign:
    | '+=' { ast.Add() }
    | '-=' { ast.Sub() }
    | '*=' { ast.Mult() }
    | '@=' { self.check_version((3, 5), "The '@' operator is", ast.MatMult()) }
    | '/=' { ast.Div() }
    | '%=' { ast.Mod() }
    | '&=' { ast.BitAnd() }
    | '|=' { ast.BitOr() }
    | '^=' { ast.BitXor() }
    | '<<=' { ast.LShift() }
    | '>>=' { ast.RShift() }
    | '**=' { ast.Pow() }
    | '//=' { ast.FloorDiv() }

return_stmt[ast.Return]:
    | 'return' a=[star_expressions] { ast.Return(value=a, LOCATIONS) }

raise_stmt[ast.Raise]:
    | 'raise' a=expression b=['from' z=expression { z }] { ast.Raise(exc=a, cause=b, LOCATIONS) }
    | 'raise' { ast.Raise(exc=None, cause=None, LOCATIONS) }

global_stmt[ast.Global]: 'global' a=','.NAME+ {
    ast.Global(names=[n.string for n in a], LOCATIONS)
}

nonlocal_stmt[ast.Nonlocal]: 'nonlocal' a=','.NAME+ {
    ast.Nonlocal(names=[n.string for n in a], LOCATIONS)
}

del_stmt[ast.Delete]:
    | 'del' a=del_targets &(';' | NEWLINE) { ast.Delete(targets=a, LOCATIONS) }
    | invalid_del_stmt

yield_stmt[ast.Expr]: y=yield_expr { ast.Expr(value=y, LOCATIONS) }

assert_stmt[ast.Assert]: 'assert' a=expression b=[',' z=expression { z }] {
    ast.Assert(test=a, msg=b, LOCATIONS)
}

import_stmt[ast.Import]:
    | invalid_import
    | import_name
    | import_from

# Import statements
# -----------------

import_name[ast.Import]: lazy="lazy"? 'import' a=dotted_as_names {
    ast.Import(names=a, is_lazy=1 if lazy else 0, LOCATIONS)
    if sys.version_info >= (3, 15) else
    self.check_version((3, 15), "Lazy imports are", ast.Import(names=a, LOCATIONS))
    if lazy else
    ast.Import(names=a, LOCATIONS)
 }

# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
import_from[ast.ImportFrom]:
    | invalid_import_from
    | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets {
        ast.ImportFrom(module=b, names=c, level=self.extract_import_level(a), is_lazy=1 if lazy else 0, LOCATIONS)
        if sys.version_info >= (3, 15) else
        self.check_version((3, 15), "Lazy imports are",
            ast.ImportFrom(module=b, names=c, level=self.extract_import_level(a), LOCATIONS))
        if lazy else
        ast.ImportFrom(module=b, names=c, level=self.extract_import_level(a), LOCATIONS)
     }
    | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets {
        ast.ImportFrom(names=b, level=self.extract_import_level(a), is_lazy=1 if lazy else 0, LOCATIONS)
        if sys.version_info >= (3, 15) else
        self.check_version((3, 15), "Lazy imports are",
            ast.ImportFrom(names=b, level=self.extract_import_level(a), LOCATIONS))
        if lazy else
        ast.ImportFrom(names=b, level=self.extract_import_level(a), LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.ImportFrom(module=None, names=b, level=self.extract_import_level(a), LOCATIONS)
     }
import_from_targets[List[ast.alias]]:
    | '(' a=import_from_as_names [','] ')' { a }
    | import_from_as_names !','
    | '*' { [ast.alias(name="*", asname=None, LOCATIONS)] }
    | invalid_import_from_targets
import_from_as_names[List[ast.alias]]:
    | a=','.import_from_as_name+ { a }
import_from_as_name[ast.alias]:
    | a=NAME b=['as' z=NAME { z.string }] { ast.alias(name=a.string, asname=b, LOCATIONS) }
dotted_as_names[List[ast.alias]]:
    | a=','.dotted_as_name+ { a }
dotted_as_name[ast.alias]:
    | a=dotted_name b=['as' z=NAME { z.string }] { ast.alias(name=a, asname=b, LOCATIONS) }
dotted_name[str]:
    | a=dotted_name '.' b=NAME { a + "." + b.string }
    | a=NAME { a.string }

# COMPOUND STATEMENTS
# ===================

# Common elements
# ---------------

block[list] (memo):
    | NEWLINE INDENT a=statements DEDENT { a }
    | simple_stmts
    | invalid_block

decorators: decorator+
decorator:
    | a=('@' f=dec_maybe_call NEWLINE { f }) { a }
    | a=('@' f=named_expression NEWLINE { f }) {
        self.check_version((3, 9), "Generic decorator are",  a)
     }
dec_maybe_call:
    | dn=dec_primary '(' z=[arguments] ')' {
        ast.Call(func=dn, args=z[0] if z else [], keywords=z[1] if z else [], LOCATIONS)
     }
    | dec_primary
dec_primary:
    | a=dec_primary '.' b=NAME { ast.Attribute(value=a, attr=b.string, ctx=Load, LOCATIONS) }
    | a=NAME { ast.Name(id=a.string, ctx=Load, LOCATIONS) }

# Class definitions
# -----------------

class_def[ast.ClassDef]:
    | a=decorators b=class_def_raw { self.set_decorators(b, a) }
    | class_def_raw

class_def_raw[ast.ClassDef]:
    | invalid_class_def_raw
    | 'class' a=NAME t=[type_params] b=['(' z=[arguments] ')' { z }] &&':' c=block {
        (
            ast.ClassDef(
                a.string,
                bases=b[0] if b else [],
                keywords=b[1] if b else [],
                body=c,
                decorator_list=[],
                type_params=t or [],
                LOCATIONS,
            )
            if sys.version_info >= (3, 12) else
            ast.ClassDef(
                a.string,
                bases=b[0] if b else [],
                keywords=b[1] if b else [],
                body=c,
                decorator_list=[],
                LOCATIONS,
            )
        )
     }

# Function definitions
# --------------------

function_def[Union[ast.FunctionDef, ast.AsyncFunctionDef]]:
    | d=decorators f=function_def_raw { self.set_decorators(f, d) }
    | f=function_def_raw {self.set_decorators(f, [])}

function_def_raw[Union[ast.FunctionDef, ast.AsyncFunctionDef]]:
    | invalid_def_raw
    | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
        (
            ast.FunctionDef(
                name=n.string,
                args=params or self.make_arguments(None, [], None, [], None),
                returns=a,
                body=b,
                type_comment=tc,
                type_params=t or [],
                LOCATIONS,
            ) if sys.version_info >= (3, 12) else
            ast.FunctionDef(
                name=n.string,
                args=params or self.make_arguments(None, [], None, [], None),
                returns=a,
                body=b,
                type_comment=tc,
                LOCATIONS,
            )
        )
     }
    | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
       (
            self.check_version(
                (3, 5),
                "Async functions are",
                ast.AsyncFunctionDef(
                    name=n.string,
                    args=params or self.make_arguments(None, [], None, [], None),
                    returns=a,
                    body=b,
                    type_comment=tc,
                    type_params=t or [],
                    LOCATIONS,
                )
            ) if sys.version_info >= (3, 12) else
            self.check_version(
                (3, 5),
                "Async functions are",
                ast.AsyncFunctionDef(
                    name=n.string,
                    args=params or self.make_arguments(None, [], None, [], None),
                    returns=a,
                    body=b,
                    type_comment=tc,
                    LOCATIONS,
                )
            )
       )
     }

# Function parameters
# -------------------

params:
    | invalid_parameters
    | parameters

parameters[ast.arguments]:
    | a=slash_no_default b=param_no_default* c=param_with_default* d=[star_etc] {
        self.check_version(
            (3, 8), "Positional only arguments are", self.make_arguments(a, [], b, c, d)
        )
     }
    | a=slash_with_default b=param_with_default* c=[star_etc] {
        self.check_version(
            (3, 8),
            "Positional only arguments are",
            self.make_arguments(None, a, None, b, c),
        )
     }
    | a=param_no_default+ b=param_with_default* c=[star_etc] {
        self.make_arguments(None, [], a, b, c)
     }
    | a=param_with_default+ b=[star_etc] {
        self.make_arguments(None, [], None, a, b)
     }
    | a=star_etc { self.make_arguments(None, [], None, None, a) }

# Some duplication here because we can't write (',' | &')'),
# which is because we don't support empty alternatives (yet).
#

slash_no_default[List[Tuple[ast.arg, None]]]:
    | a=param_no_default+ '/' ',' { [(p, None) for p in a] }
    | a=param_no_default+ '/' &')' { [(p, None) for p in a] }
slash_with_default[List[Tuple[ast.arg, Any]]]:
    | a=param_no_default* b=param_with_default+ '/' ',' { ([(p, None) for p in a] if a else []) + b }
    | a=param_no_default* b=param_with_default+ '/' &')' { ([(p, None) for p in a] if a else []) + b }

star_etc[Tuple[Optional[ast.arg], List[Tuple[ast.arg, Any]], Optional[ast.arg]]]:
    | invalid_star_etc
    | '*' a=param_no_default b=param_maybe_default* c=[kwds] { (a, b, c) }
    | '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds] { (a, b, c) }
    | '*' ',' b=param_maybe_default+ c=[kwds] { (None, b, c) }
    | a=kwds { (None, [], a) }

kwds[ast.arg]:
    | invalid_kwds
    | '**' a=param_no_default { a }

# One parameter.  This *includes* a following comma and type comment.
#
# There are three styles:
# - No default
# - With default
# - Maybe with default
#
# There are two alternative forms of each, to deal with type comments:
# - Ends in a comma followed by an optional type comment
# - No comma, optional type comment, must be followed by close paren
# The latter form is for a final parameter without trailing comma.
#

param_no_default[ast.arg]:
    | a=param ',' tc=TYPE_COMMENT? { self.set_arg_type_comment(a, tc) }
    | a=param tc=TYPE_COMMENT? &')' { self.set_arg_type_comment(a, tc) }
param_no_default_star_annotation[ast.arg]:
    | a=param_star_annotation ',' tc=TYPE_COMMENT? { self.set_arg_type_comment(a, tc) }
    | a=param_star_annotation tc=TYPE_COMMENT? &')' { self.set_arg_type_comment(a, tc) }
param_with_default[Tuple[ast.arg, Any]]:
    | a=param c=default ',' tc=TYPE_COMMENT? { (self.set_arg_type_comment(a, tc), c) }
    | a=param c=default tc=TYPE_COMMENT? &')' { (self.set_arg_type_comment(a, tc), c) }
param_maybe_default[Tuple[ast.arg, Any]]:
    | a=param c=default? ',' tc=TYPE_COMMENT? { (self.set_arg_type_comment(a, tc), c) }
    | a=param c=default? tc=TYPE_COMMENT? &')' { (self.set_arg_type_comment(a, tc), c) }
param: a=NAME b=annotation? { ast.arg(arg=a.string, annotation=b, LOCATIONS) }
param_star_annotation: a=NAME b=star_annotation {
    ast.arg(arg=a.string, annotation=b, LOCATIONS)
 }
annotation: ':' a=expression { a }
star_annotation: ':' a=star_expression { a }
default: '=' a=expression { a } | invalid_default

# If statement
# ------------

if_stmt[ast.If]:
    | invalid_if_stmt
    | 'if' a=named_expression ':' b=block c=elif_stmt { ast.If(test=a, body=b, orelse=c or [], LOCATIONS) }
    | 'if' a=named_expression ':' b=block c=[else_block] { ast.If(test=a, body=b, orelse=c or [], LOCATIONS) }
elif_stmt[List[ast.If]]:
    | invalid_elif_stmt
    | 'elif' a=named_expression ':' b=block c=elif_stmt { [ast.If(test=a, body=b, orelse=c, LOCATIONS)] }
    | 'elif' a=named_expression ':' b=block c=[else_block] { [ast.If(test=a, body=b, orelse=c or [], LOCATIONS)] }
else_block[list]:
    | invalid_else_stmt
    | 'else' &&':' b=block { b }

# While statement
# ---------------

while_stmt[ast.While]:
    | invalid_while_stmt
    | 'while' a=named_expression ':' b=block c=[else_block] {
        ast.While(test=a, body=b, orelse=c or [], LOCATIONS)
     }

# For statement
# -------------

for_stmt[Union[ast.For, ast.AsyncFor]]:
    | invalid_for_stmt
    | 'for' t=star_targets 'in' ~ ex=star_expressions &&':' tc=[TYPE_COMMENT] b=block el=[else_block] {
        ast.For(target=t, iter=ex, body=b, orelse=el or [], type_comment=tc, LOCATIONS) }
    | 'async' 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
        self.check_version(
            (3, 5),
            "Async for loops are",
            ast.AsyncFor(target=t, iter=ex, body=b, orelse=el or [], type_comment=tc, LOCATIONS)) }
    | invalid_for_target

# With statement
# --------------

with_stmt[Union[ast.With, ast.AsyncWith]]:
    | invalid_with_stmt_indent
    | 'with' '(' a=','.with_item+ ','? ')' ':' b=block {
        self.check_version(
           (3, 9),
           "Parenthesized with items",
            ast.With(items=a, body=b, LOCATIONS)
        )
     }
    | 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
        ast.With(items=a, body=b, type_comment=tc, LOCATIONS)
     }
    | 'async' 'with' '(' a=','.with_item+ ','? ')' ':' b=block {
       self.check_version(
           (3, 9),
           "Parenthesized with items",
           ast.AsyncWith(items=a, body=b, LOCATIONS)
        )
     }
    | 'async' 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
       self.check_version(
           (3, 5),
           "Async with statements are",
           ast.AsyncWith(items=a, body=b, type_comment=tc, LOCATIONS)
        )
     }
    | invalid_with_stmt

with_item[ast.withitem]:
    | e=expression 'as' t=star_target &(',' | ')' | ':') {
        ast.withitem(context_expr=e, optional_vars=t)
     }
    | invalid_with_item
    | e=expression { ast.withitem(context_expr=e, optional_vars=None) }

# Try statement
# -------------

try_stmt[ast.Try]:
    | invalid_try_stmt
    | 'try' &&':' b=block f=finally_block {
        ast.Try(body=b, handlers=[], orelse=[], finalbody=f, LOCATIONS)
     }
    | 'try' &&':' b=block ex=except_block+ el=[else_block] f=[finally_block] {
        ast.Try(body=b, handlers=ex, orelse=el or [], finalbody=f or [], LOCATIONS)
     }
    | 'try' &&':' b=block ex=except_star_block+ el=[else_block] f=[finally_block] {
        self.check_version(
            (3, 11),
            "Exception groups are",
            (
                ast.TryStar(body=b, handlers=ex, orelse=el or [], finalbody=f or [], LOCATIONS)
                if sys.version_info >= (3, 11)
                else None
            )
        )
     }

# Except statement
# ----------------

except_block[ast.ExceptHandler]:
    | invalid_except_stmt_indent
    | invalid_except_stmt
    | 'except' e=expression t=['as' z=NAME { z.string }] ':' b=block {
        ast.ExceptHandler(type=e, name=t, body=b, LOCATIONS) }
    | 'except' ':' b=block { ast.ExceptHandler(type=None, name=None, body=b, LOCATIONS) }
    | 'except' e=expressions ':' b=block {
        ast.ExceptHandler(type=e, name=None, body=b, LOCATIONS)
        if self.py_version >= (3, 14)
        else None }
except_star_block[ast.ExceptHandler]:
    | invalid_except_star_stmt_indent
    | invalid_except_stmt
    | 'except' '*' e=expression t=['as' z=NAME { z.string }] ':' b=block {
        ast.ExceptHandler(type=e, name=t, body=b, LOCATIONS)
     }
    | 'except' '*' e=expressions ':' b=block {
        ast.ExceptHandler(type=e, name=None, body=b, LOCATIONS)
        if self.py_version >= (3, 14)
        else None
     }
finally_block[list]:
    | invalid_finally_stmt
    | 'finally' &&':' a=block { a }

# Match statement
# ---------------

# We cannot do version checks here since the production will occur after any other
# production which will have failed since the ast module does not have the right nodes.
match_stmt["ast.Match"]:
    | "match" subject=subject_expr ':' NEWLINE INDENT cases=case_block+ DEDENT {
        ast.Match(subject=subject, cases=cases, LOCATIONS)
     }
    | invalid_match_stmt

# Version checking here allows to avoid tracking down every single possible production
subject_expr:
    | value=star_named_expression ',' values=star_named_expressions? {
        self.check_version(
            (3, 10),
            "Pattern matching is",
            ast.Tuple(elts=[value] + (values or []), ctx=Load, LOCATIONS)
        )
     }
    | e=named_expression { self.check_version((3, 10), "Pattern matching is", e)}

case_block["ast.match_case"]:
    | invalid_case_block
    | "case" pattern=patterns guard=guard? ':' body=block {
        ast.match_case(pattern=pattern, guard=guard, body=body)
     }

guard: 'if' guard=named_expression { guard }

patterns:
    | patterns=open_sequence_pattern {
        ast.MatchSequence(patterns=patterns, LOCATIONS)
     }
    | pattern

pattern:
    | as_pattern
    | or_pattern

as_pattern["ast.MatchAs"]:
    | pattern=or_pattern 'as' target=pattern_capture_target {
        ast.MatchAs(pattern=pattern, name=target, LOCATIONS)
     }
    | invalid_as_pattern

or_pattern["ast.MatchOr"]:
    | patterns='|'.closed_pattern+ {
        ast.MatchOr(patterns=patterns, LOCATIONS) if len(patterns) > 1 else patterns[0]
     }

closed_pattern:
    | literal_pattern
    | capture_pattern
    | wildcard_pattern
    | value_pattern
    | group_pattern
    | sequence_pattern
    | mapping_pattern
    | class_pattern

# Literal patterns are used for equality and identity constraints
literal_pattern:
    | value=signed_number !('+' | '-') { ast.MatchValue(value=value, LOCATIONS) }
    | value=complex_number { ast.MatchValue(value=value, LOCATIONS) }
    | value=strings { ast.MatchValue(value=value, LOCATIONS) }
    | 'None' { ast.MatchSingleton(value=None, LOCATIONS) }
    | 'True' { ast.MatchSingleton(value=True, LOCATIONS) }
    | 'False' { ast.MatchSingleton(value=False, LOCATIONS) }

# Literal expressions are used to restrict permitted mapping pattern keys
literal_expr:
    | signed_number !('+' | '-')
    | complex_number
    | strings
    | 'None' { ast.Constant(value=None, LOCATIONS) }
    | 'True' { ast.Constant(value=True, LOCATIONS) }
    | 'False' { ast.Constant(value=False, LOCATIONS) }

complex_number:
    | real=signed_real_number '+' imag=imaginary_number {
        ast.BinOp(left=real, op=ast.Add(), right=imag, LOCATIONS)
     }
    | real=signed_real_number '-' imag=imaginary_number  {
        ast.BinOp(left=real, op=ast.Sub(), right=imag, LOCATIONS)
     }

signed_number:
    | a=NUMBER { ast.Constant(value=ast.literal_eval(a.string), LOCATIONS) }
    | '-' a=NUMBER {
        ast.UnaryOp(
            op=ast.USub(),
            operand=ast.Constant(
                value=ast.literal_eval(a.string),
                lineno=a.start[0],
                col_offset=a.start[1],
                end_lineno=a.end[0],
                end_col_offset=a.end[1]
            ),
            LOCATIONS,
        )
     }

signed_real_number:
    | real_number
    | '-' real=real_number { ast.UnaryOp(op=ast.USub(), operand=real, LOCATIONS) }

real_number[ast.Constant]:
    | real=NUMBER { ast.Constant(value=self.ensure_real(real), LOCATIONS) }

imaginary_number[ast.Constant]:
    | imag=NUMBER { ast.Constant(value=self.ensure_imaginary(imag), LOCATIONS) }

capture_pattern:
    | target=pattern_capture_target {
        ast.MatchAs(pattern=None, name=target, LOCATIONS)
     }

pattern_capture_target[str]:
    | !"_" name=NAME !('.' | '(' | '=') { name.string }

wildcard_pattern["ast.MatchAs"]:
    | "_" { ast.MatchAs(pattern=None, name=None, LOCATIONS) }

value_pattern["ast.MatchValue"]:
    | attr=attr !('.' | '(' | '=') { ast.MatchValue(value=attr, LOCATIONS) }

attr[ast.Attribute]:
    | value=name_or_attr '.' attr=NAME {
        ast.Attribute(value=value, attr=attr.string, ctx=Load, LOCATIONS)
     }

name_or_attr:
    | attr
    | name=NAME { ast.Name(id=name.string, ctx=Load, LOCATIONS) }

group_pattern:
    | '(' pattern=pattern ')' { pattern }

sequence_pattern["ast.MatchSequence"]:
    | '[' patterns=maybe_sequence_pattern? ']' { ast.MatchSequence(patterns=patterns or [], LOCATIONS) }
    | '(' patterns=open_sequence_pattern? ')' { ast.MatchSequence(patterns=patterns or [], LOCATIONS) }

open_sequence_pattern:
    | pattern=maybe_star_pattern ',' patterns=maybe_sequence_pattern? {
        [pattern] + (patterns or [])
     }

maybe_sequence_pattern:
    | patterns=','.maybe_star_pattern+ ','? { patterns }

maybe_star_pattern:
    | star_pattern
    | pattern

star_pattern:
    | '*' target=pattern_capture_target { ast.MatchStar(name=target, LOCATIONS) }
    | '*' wildcard_pattern { ast.MatchStar(name=None, LOCATIONS) }

mapping_pattern:
    | '{' '}' { ast.MatchMapping(keys=[], patterns=[], rest=None, LOCATIONS) }
    | '{' rest=double_star_pattern ','? '}' {
        ast.MatchMapping(keys=[], patterns=[], rest=rest, LOCATIONS) }
    | '{' items=items_pattern ',' rest=double_star_pattern ','? '}' {
        ast.MatchMapping(
            keys=[k for k,_ in items],
            patterns=[p for _, p in items],
            rest=rest,
            LOCATIONS,
        )
     }
    | '{' items=items_pattern ','? '}' {
        ast.MatchMapping(
            keys=[k for k,_ in items],
            patterns=[p for _, p in items],
            rest=None,
            LOCATIONS,
        )
     }

items_pattern:
    | ','.key_value_pattern+

key_value_pattern:
    | key=(literal_expr | attr) ':' pattern=pattern { (key, pattern) }

double_star_pattern:
    | '**' target=pattern_capture_target { target }

class_pattern["ast.MatchClass"]:
    | cls=name_or_attr '(' ')' {
        ast.MatchClass(cls=cls, patterns=[], kwd_attrs=[], kwd_patterns=[], LOCATIONS)
     }
    | cls=name_or_attr '(' patterns=positional_patterns ','? ')' {
        ast.MatchClass(cls=cls, patterns=patterns, kwd_attrs=[], kwd_patterns=[], LOCATIONS)
     }
    | cls=name_or_attr '(' keywords=keyword_patterns ','? ')' {
        ast.MatchClass(
            cls=cls,
            patterns=[],
            kwd_attrs=[k for k, _ in keywords],
            kwd_patterns=[p for _, p in keywords],
            LOCATIONS,
        )
     }
    | cls=name_or_attr '(' patterns=positional_patterns ',' keywords=keyword_patterns ','? ')' {
        ast.MatchClass(
            cls=cls,
            patterns=patterns,
            kwd_attrs=[k for k, _ in keywords],
            kwd_patterns=[p for _, p in keywords],
            LOCATIONS,
        )
     }
    | invalid_class_pattern

positional_patterns:
    | args=','.pattern+ { args }

keyword_patterns:
    | ','.keyword_pattern+

keyword_pattern:
    | arg=NAME '=' value=pattern { (arg.string, value) }

# Type statement
# ---------------

type_alias["ast.TypeAlias"]:
    | "type" n=NAME t=[type_params] '=' b=expression {
        self.check_version(
            (3, 12),
            "Type statement is",
            (
                ast.TypeAlias(
                    name=ast.Name(
                        id=n.string,
                        ctx=Store,
                        lineno=n.start[0],
                        col_offset=n.start[1],
                        end_lineno=n.end[0],
                        end_col_offset=n.end[1],
                    ),
                    type_params=t or [],
                    value=b,
                    LOCATIONS
                )
                if sys.version_info >= (3, 12)
                else None
            )
        )
     }

# Type parameter declaration
# --------------------------

type_params[list]: '[' t=type_param_seq  ']' {
    self.check_version(
        (3, 12),
        "Type parameter lists are",
        t
    )
 }

type_param_seq: a=','.type_param+ [','] { a }

type_param (memo):
    | a=NAME b=[type_param_bound] d=[type_param_default] {
        type_parameter(self, "TypeVar", a.string, b, d, LOCATIONS)
     }
    | '*' a=NAME colon=":" e=expression {
        self.raise_syntax_error_starting_from(
            "cannot use constraints with TypeVarTuple"
            if isinstance(e, ast.Tuple)
            else "cannot use bound with TypeVarTuple",
            colon
        )
     }
    | '*' a=NAME d=[type_param_starred_default] {
        type_parameter(self, "TypeVarTuple", a.string, None, d, LOCATIONS)
     }
    | '**' a=NAME colon=":" e=expression {
        self.raise_syntax_error_starting_from(
            "cannot use constraints with ParamSpec"
            if isinstance(e, ast.Tuple)
            else "cannot use bound with ParamSpec",
            colon
        )
     }
    | '**' a=NAME d=[type_param_default] {
        type_parameter(self, "ParamSpec", a.string, None, d, LOCATIONS)
     }

type_param_bound: ":" e=expression { e }
type_param_default: "=" e=expression { self.check_version((3, 13), "Type parameter defaults are", e) }
type_param_starred_default: "=" e=star_expression { self.check_version((3, 13), "Type parameter defaults are", e) }

# EXPRESSIONS
# -----------

factor_with_units[UnitsExpr]: f=factor u=units {UnitsExpr(f, u, LOCATIONS)}

units[ast.Constant]: u=units_ {ast.Constant(value=u, LOCATIONS)}

units_[str]:
    | a=units_ '/' c=units_group_ {a + '/' + c}
    | a=units_ '*' c=units_group_ {a + '*' + c}
    | a=units_ b=units_group_ {a + ' ' + b}
    | a=units_ '**' sign='-'? c=NUMBER {a + '**' + (sign.string if sign is not None else "") + c.string}
    | a=NAME {a.string}

units_group_[str]:
    | '(' a=units_ ')' {'(' + a + ')'}
    | units_


expressions:
    | a=expression b=(',' c=expression { c })+ [','] {
        ast.Tuple(elts=[a] + b, ctx=Load, LOCATIONS) }
    | a=expression ',' { ast.Tuple(elts=[a], ctx=Load, LOCATIONS) }
    | expression

expression (memo):
    | factor_with_units
    | invalid_expression
    | invalid_legacy_expression
    | a=disjunction 'if' b=disjunction 'else' c=expression {
        ast.IfExp(body=a, test=b, orelse=c, LOCATIONS)
     }
    | disjunction
    | lambdef

yield_expr:
    | 'yield' 'from' a=expression { ast.YieldFrom(value=a, LOCATIONS) }
    | 'yield' a=[star_expressions] { ast.Yield(value=a, LOCATIONS) }

star_expressions:
    | a=star_expression b=(',' c=star_expression { c })+ [','] {
        ast.Tuple(elts=[a] + b, ctx=Load, LOCATIONS) }
    | a=star_expression ',' { ast.Tuple(elts=[a], ctx=Load, LOCATIONS) }
    | star_expression

star_expression (memo):
    | '*' a=bitwise_or { ast.Starred(value=a, ctx=Load, LOCATIONS) }
    | expression

star_named_expressions: a=','.star_named_expression+ [','] { a }

star_named_expression:
    | '*' a=bitwise_or { ast.Starred(value=a, ctx=Load, LOCATIONS) }
    | named_expression

assignment_expression:
    | a=NAME ':=' ~ b=expression {
        self.check_version(
            (3, 8),
            "The ':=' operator is",
            ast.NamedExpr(
                target=ast.Name(
                    id=a.string,
                    ctx=Store,
                    lineno=a.start[0],
                    col_offset=a.start[1],
                    end_lineno=a.end[0],
                    end_col_offset=a.end[1]
                ),
                value=b,
                LOCATIONS,
            )
        )
     }

named_expression:
    | assignment_expression
    | invalid_named_expression
    | a=expression !':=' { a }

disjunction (memo):
    | a=conjunction b=('or' c=conjunction { c })+ { ast.BoolOp(op=ast.Or(), values=[a] + b, LOCATIONS) }
    | conjunction

conjunction (memo):
    | a=inversion b=('and' c=inversion { c })+ { ast.BoolOp(op=ast.And(), values=[a] + b, LOCATIONS) }
    | inversion

inversion (memo):
    | 'not' a=inversion { ast.UnaryOp(op=ast.Not(), operand=a, LOCATIONS) }
    | comparison

# Comparisons operators
# ---------------------

comparison:
    | a=bitwise_or b=compare_op_bitwise_or_pair+ {
        ast.Compare(left=a, ops=self.get_comparison_ops(b), comparators=self.get_comparators(b), LOCATIONS)
     }
    | bitwise_or

# Make a tuple of operator and comparator
compare_op_bitwise_or_pair:
    | eq_bitwise_or
    | noteq_bitwise_or
    | lte_bitwise_or
    | lt_bitwise_or
    | gte_bitwise_or
    | gt_bitwise_or
    | notin_bitwise_or
    | in_bitwise_or
    | isnot_bitwise_or
    | is_bitwise_or

eq_bitwise_or: '==' a=bitwise_or { (ast.Eq(), a) }
# Do not support the Barry as BDFL <> for not eq
noteq_bitwise_or[tuple]:
    | '!=' a=bitwise_or { (ast.NotEq(), a) }
lte_bitwise_or: '<=' a=bitwise_or { (ast.LtE(), a) }
lt_bitwise_or: '<' a=bitwise_or { (ast.Lt(), a) }
gte_bitwise_or: '>=' a=bitwise_or { (ast.GtE(), a) }
gt_bitwise_or: '>' a=bitwise_or { (ast.Gt(), a) }
notin_bitwise_or: 'not' 'in' a=bitwise_or { (ast.NotIn(), a) }
in_bitwise_or: 'in' a=bitwise_or { (ast.In(), a) }
isnot_bitwise_or: 'is' 'not' a=bitwise_or { (ast.IsNot(), a) }
is_bitwise_or: 'is' a=bitwise_or { (ast.Is(), a) }

# Logical operators
# -----------------

bitwise_or:
    | f=factor u=units {
        self.raise_syntax_error_known_location(
            "Units must be parenthesized when mixed with operators",
            u,
        )
     }
    | a=bitwise_or '|' b=bitwise_xor { ast.BinOp(left=a, op=ast.BitOr(), right=b, LOCATIONS) }
    | bitwise_xor

bitwise_xor:
    | a=bitwise_xor '^' b=bitwise_and { ast.BinOp(left=a, op=ast.BitXor(), right=b, LOCATIONS) }
    | bitwise_and

bitwise_and:
    | a=bitwise_and '&' b=shift_expr { ast.BinOp(left=a, op=ast.BitAnd(), right=b, LOCATIONS) }
    | shift_expr

shift_expr:
    | a=shift_expr '<<' b=sum { ast.BinOp(left=a, op=ast.LShift(), right=b, LOCATIONS) }
    | a=shift_expr '>>' b=sum { ast.BinOp(left=a, op=ast.RShift(), right=b, LOCATIONS) }
    | sum

# Arithmetic operators
# --------------------

sum:
    | a=sum '+' b=term { ast.BinOp(left=a, op=ast.Add(), right=b, LOCATIONS) }
    | a=sum '-' b=term { ast.BinOp(left=a, op=ast.Sub(), right=b, LOCATIONS) }
    | term

term:
    | a=term '*' b=factor { ast.BinOp(left=a, op=ast.Mult(), right=b, LOCATIONS) }
    | a=term '/' b=factor { ast.BinOp(left=a, op=ast.Div(), right=b, LOCATIONS) }
    | a=term '//' b=factor { ast.BinOp(left=a, op=ast.FloorDiv(), right=b, LOCATIONS) }
    | a=term '%' b=factor { ast.BinOp(left=a, op=ast.Mod(), right=b, LOCATIONS) }
    | a=term '@' b=factor {
        self.check_version((3, 5), "The '@' operator is", ast.BinOp(left=a, op=ast.MatMult(), right=b, LOCATIONS))
     }
    | factor

factor (memo):
    | '+' a=factor { ast.UnaryOp(op=ast.UAdd(), operand=a, LOCATIONS) }
    | '-' a=factor { ast.UnaryOp(op=ast.USub(), operand=a, LOCATIONS) }
    | '~' a=factor { ast.UnaryOp(op=ast.Invert(), operand=a, LOCATIONS) }
    | power

power:
    | a=await_primary '**' b=factor { ast.BinOp(left=a, op=ast.Pow(), right=b, LOCATIONS) }
    | await_primary

# Primary elements
# ----------------

# Primary elements are things like "obj.something.something", "obj[something]", "obj(something)", "obj" ...

await_primary (memo):
    | 'await' a=primary { self.check_version((3, 5), "Await expressions are", ast.Await(a, LOCATIONS)) }
    | primary

primary:
    | a=primary '.' b=NAME { ast.Attribute(value=a, attr=b.string, ctx=Load, LOCATIONS) }
    | a=primary b=genexp { ast.Call(func=a, args=[b], keywords=[], LOCATIONS) }
    | a=primary '(' b=[arguments] ')' {
        ast.Call(
            func=a,
            args=b[0] if b else [],
            keywords=b[1] if b else [],
            LOCATIONS,
        )
     }
    | a=primary '[' b=slices ']' { ast.Subscript(value=a, slice=b, ctx=Load, LOCATIONS) }
    | atom

slices:
    | a=slice !',' { a }
    | a=','.(slice | starred_expression)+ [','] {
        ast.Tuple(elts=a, ctx=Load, LOCATIONS)
        if sys.version_info >= (3, 9) else
        (
            ast.ExtSlice(dims=a, LOCATIONS)
            if any(isinstance(e, ast.Slice) for e in a) else
            ast.Index(value=ast.Tuple(elts=[e.value for e in a], ctx=Load, LOCATIONS), LOCATIONS)
        )
     }

slice:
    | a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] {
        ast.Slice(lower=a, upper=b, step=c, LOCATIONS)
     }
    | a=named_expression {
        a
        if sys.version_info >= (3, 9) or isinstance(a, ast.Slice) else
        ast.Index(
            value=a,
            lineno=a.lineno,
            col_offset=a.col_offset,
            end_lineno=a.end_lineno,
            end_col_offset=a.end_col_offset
        )
     }

atom:
    | a=NAME { ast.Name(id=a.string, ctx=Load, LOCATIONS) }
    | 'True' {
        ast.Constant(value=True, LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.Constant(value=True, kind=None, LOCATIONS)
     }
    | 'False' {
        ast.Constant(value=False, LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.Constant(value=False, kind=None, LOCATIONS)
     }
    | 'None' {
        ast.Constant(value=None, LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.Constant(value=None, kind=None, LOCATIONS)
     }
    | &(STRING|FSTRING_START|tstring_start_token) strings
    | a=NUMBER {
        ast.Constant(value=ast.literal_eval(a.string), LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.Constant(value=ast.literal_eval(a.string), kind=None, LOCATIONS)
     }
    | &'(' (tuple | group | genexp)
    | &'[' (list | listcomp)
    | &'{' (dict | set | dictcomp | setcomp)
    | '...' {
        ast.Constant(value=Ellipsis, LOCATIONS)
        if sys.version_info >= (3, 9) else
        ast.Constant(value=Ellipsis, kind=None, LOCATIONS)
     }

group:
    | '(' a=(yield_expr | named_expression) ')' { a }
    | invalid_group


# Lambda functions
# ----------------

lambdef:
    | 'lambda' a=[lambda_params] ':' b=expression {
        ast.Lambda(args=a or self.make_arguments(None, [], None, [], (None, [], None)), body=b, LOCATIONS)
     }

lambda_params:
    | invalid_lambda_parameters
    | lambda_parameters

# lambda_parameters etc. duplicates parameters but without annotations
# or type comments, and if there's no comma after a parameter, we expect
# a colon, not a close parenthesis.  (For more, see parameters above.)
#
lambda_parameters[ast.arguments]:
    | a=lambda_slash_no_default b=lambda_param_no_default* c=lambda_param_with_default* d=[lambda_star_etc] {
        self.make_arguments(a, [], b, c, d)
     }
    | a=lambda_slash_with_default b=lambda_param_with_default* c=[lambda_star_etc] {
        self.make_arguments(None, a, None, b, c)
     }
    | a=lambda_param_no_default+ b=lambda_param_with_default* c=[lambda_star_etc] {
        self.make_arguments(None, [], a, b, c)
     }
    | a=lambda_param_with_default+ b=[lambda_star_etc] {
        self.make_arguments(None, [], None, a, b)
     }
    | a=lambda_star_etc { self.make_arguments(None, [], None, [], a) }

lambda_slash_no_default[List[Tuple[ast.arg, None]]]:
    | a=lambda_param_no_default+ '/' ',' { [(p, None) for p in a] }
    | a=lambda_param_no_default+ '/' &':' { [(p, None) for p in a] }

lambda_slash_with_default[List[Tuple[ast.arg, Any]]]:
    | a=lambda_param_no_default* b=lambda_param_with_default+ '/' ',' { ([(p, None) for p in a] if a else []) + b }
    | a=lambda_param_no_default* b=lambda_param_with_default+ '/' &':' { ([(p, None) for p in a] if a else []) + b }

lambda_star_etc[Tuple[Optional[ast.arg], List[Tuple[ast.arg, Any]], Optional[ast.arg]]]:
    | invalid_lambda_star_etc
    | '*' a=lambda_param_no_default b=lambda_param_maybe_default* c=[lambda_kwds] {
       (a, b, c) }
    | '*' ',' b=lambda_param_maybe_default+ c=[lambda_kwds] {
        (None, b, c) }
    | a=lambda_kwds { (None, [], a) }

lambda_kwds[ast.arg]:
    | invalid_lambda_kwds
    | '**' a=lambda_param_no_default { a }

lambda_param_no_default[ast.arg]:
    | a=lambda_param ',' { a }
    | a=lambda_param &':' { a }

lambda_param_with_default[Tuple[ast.arg, Any]]:
    | a=lambda_param c=default ',' { (a, c) }
    | a=lambda_param c=default &':' { (a, c) }
lambda_param_maybe_default[Tuple[ast.arg, Any]]:
    | a=lambda_param c=default? ',' { (a, c) }
    | a=lambda_param c=default? &':' { (a, c) }
lambda_param[ast.arg]: a=NAME {
    ast.arg(arg=a.string, annotation=None, LOCATIONS)
    if sys.version_info >= (3, 9) else
    ast.arg(arg=a.string, annotation=None, type_comment=None, LOCATIONS)
 }

# LITERALS
# ========

fstring_mid:
    | fstring_replacement_field
    | t=FSTRING_MIDDLE { ast.Constant(value=t.string, LOCATIONS) }
fstring_replacement_field:
    | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
        formatted_field(ast.FormattedValue(
            value=a,
            conversion=(
                conversion
                if conversion is not None else
                (ord('r') if debug_expr else -1)
            ),
            format_spec=format,
            LOCATIONS
        ), debug_expr)
     }
    | invalid_replacement_field
fstring_conversion[int]:
    | conv_token="!" conv=NAME { self.check_fstring_conversion(conv_token, conv) }
fstring_full_format_spec:
    | ':' spec=fstring_format_spec* {
        ast.JoinedStr(
            values=spec if spec and (len(spec) > 1 or spec[0].value) else [],
            LOCATIONS,
        )
     }
fstring_format_spec:
    | t=FSTRING_MIDDLE { ast.Constant(value=t.string, LOCATIONS) }
    | fstring_replacement_field
fstring:
    | a=FSTRING_START b=fstring_mid* c=FSTRING_END {
        interpolated_string(self, a, b, c)
     }

tstring_start_token: !ENDMARKER { self.expect("TSTRING_START") }
tstring_middle_token: !ENDMARKER { self.expect("TSTRING_MIDDLE") }
tstring_end_token: !ENDMARKER { self.expect("TSTRING_END") }

tstring_mid:
    | tstring_replacement_field
    | t=tstring_middle_token { ast.Constant(value=t.string, LOCATIONS) }
tstring_replacement_field:
    | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[tstring_full_format_spec] rbrace='}' {
        formatted_field(ast.FormattedValue(
            value=a,
            conversion=(
                conversion
                if conversion is not None else
                (ord('r') if debug_expr else -1)
            ),
            format_spec=format,
            LOCATIONS
        ), debug_expr)
     }
    | invalid_replacement_field
tstring_full_format_spec:
    | ':' spec=tstring_format_spec* {
        ast.JoinedStr(
            values=spec if spec and (len(spec) > 1 or spec[0].value) else [],
            LOCATIONS,
        )
     }
tstring_format_spec:
    | t=tstring_middle_token { ast.Constant(value=t.string, LOCATIONS) }
    | tstring_replacement_field
tstring:
    | a=tstring_start_token b=tstring_mid* c=tstring_end_token {
        interpolated_string(self, a, b, c)
     }

strings (memo):
    | a=tstring+ { self.check_version((3, 14), "t-strings are", concatenate_templates(a)) }
    | a=(fstring|STRING)+ {
    self.concatenate_strings(a) if sys.version_info >= (3, 12) else self.generate_ast_for_string(a)
 }

list[ast.List]:
    | '[' a=[star_named_expressions] ']' { ast.List(elts=a or [], ctx=Load, LOCATIONS) }

tuple[ast.Tuple]:
    | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { [y] + (z or []) } ] ')' {
        ast.Tuple(elts=a or [], ctx=Load, LOCATIONS)
     }

set[ast.Set]: '{' a=star_named_expressions '}' { ast.Set(elts=a, LOCATIONS) }

# Dicts
# -----

dict[ast.Dict]:
    | '{' a=[double_starred_kvpairs] '}' {
        ast.Dict(keys=[kv[0] for kv in (a or [])], values=[kv[1] for kv in (a or [])], LOCATIONS)
     }
    | '{' invalid_double_starred_kvpairs '}'

double_starred_kvpairs[list]: a=','.double_starred_kvpair+ [','] { a }

double_starred_kvpair:
    | '**' a=bitwise_or { (None, a) }
    | kvpair

kvpair[tuple]: a=expression ':' b=expression { (a, b) }

# Comprehensions & Generators
# ---------------------------

for_if_clauses[List[ast.comprehension]]:
    | a=for_if_clause+ { a }

for_if_clause[ast.comprehension]:
    | 'async' 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* {
        self.check_version(
            (3, 6),
            "Async comprehensions are",
            ast.comprehension(target=a, iter=b, ifs=c, is_async=1)
        )
     }
    | 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* {
       ast.comprehension(target=a, iter=b, ifs=c, is_async=0) }
    | invalid_for_target

listcomp[ast.ListComp]:
    | invalid_comprehension
    | '[' a=named_expression b=for_if_clauses ']' { ast.ListComp(elt=a, generators=b, LOCATIONS) }
    | '[' a=starred_expression b=for_if_clauses ']' {
        ast.ListComp(elt=a, generators=b, LOCATIONS)
        if self.py_version >= (3, 15)
        else None }

setcomp[ast.SetComp]:
    | invalid_comprehension
    | '{' a=named_expression b=for_if_clauses '}' { ast.SetComp(elt=a, generators=b, LOCATIONS) }
    | '{' a=starred_expression b=for_if_clauses '}' {
        ast.SetComp(elt=a, generators=b, LOCATIONS)
        if self.py_version >= (3, 15)
        else None }

genexp[ast.GeneratorExp]:
    | invalid_comprehension
    | '(' a=(assignment_expression | expression !':=') b=for_if_clauses ')' {
        ast.GeneratorExp(elt=a, generators=b, LOCATIONS) }
    | '(' a=starred_expression b=for_if_clauses ')' {
        ast.GeneratorExp(elt=a, generators=b, LOCATIONS)
        if self.py_version >= (3, 15)
        else None }

dictcomp[ast.DictComp]:
    | invalid_dict_comprehension
    | '{' a=kvpair b=for_if_clauses '}' {
        ast.DictComp(key=a[0], value=a[1], generators=b, LOCATIONS) }
    | '{' '**' a=bitwise_or b=for_if_clauses '}' {
        ast.DictComp(key=a, value=None, generators=b, LOCATIONS)
        if self.py_version >= (3, 15)
        else None }

# FUNCTION CALL ARGUMENTS
# =======================

arguments[Tuple[list, list]] (memo):
    | a=args [','] &')' { a }
    | invalid_arguments

args[Tuple[list, list]]:
    | a=','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ b=[',' k=kwargs {k}] {
        (a + ([e for e in b if isinstance(e, ast.Starred)] if b else []),
         ([e for e in b if not isinstance(e, ast.Starred)] if b else [])
        )
     }
    | a=kwargs {
        ([e for e in a if isinstance(e, ast.Starred)],
         [e for e in a if not isinstance(e, ast.Starred)])
    }

kwargs[list]:
    | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { a + b }
    | ','.kwarg_or_starred+
    | ','.kwarg_or_double_starred+

starred_expression:
    | invalid_starred_expression
    | '*' a=expression { ast.Starred(value=a, ctx=Load, LOCATIONS) }
    | '*' { self.raise_syntax_error("Invalid star expression") }

kwarg_or_starred:
    | invalid_kwarg
    | a=NAME '=' b=expression { ast.keyword(arg=a.string, value=b, LOCATIONS) }
    | a=starred_expression { a }

kwarg_or_double_starred:
    | invalid_kwarg
    | a=NAME '=' b=expression { ast.keyword(arg=a.string, value=b, LOCATIONS) }   # XXX Unreachable
    | '**' a=expression { ast.keyword(arg=None, value=a, LOCATIONS) }

# ASSIGNMENT TARGETS
# ==================

# Generic targets
# ---------------

# NOTE: star_targets may contain *bitwise_or, targets may not.
star_targets:
    | a=star_target !',' { a }
    | a=star_target b=(',' c=star_target { c })* [','] {
        ast.Tuple(elts=[a] + b, ctx=Store, LOCATIONS)
     }

star_targets_list_seq[list]: a=','.star_target+ [','] { a }

star_targets_tuple_seq[list]:
    | a=star_target b=(',' c=star_target { c })+ [','] { [a] + b }
    | a=star_target ',' { [a] }

star_target (memo):
    | '*' a=(!'*' star_target) {
        ast.Starred(value=self.set_expr_context(a, Store), ctx=Store, LOCATIONS)
     }
    | target_with_star_atom

target_with_star_atom (memo):
    | a=t_primary '.' b=NAME !t_lookahead { ast.Attribute(value=a, attr=b.string, ctx=Store, LOCATIONS) }
    | a=t_primary '[' b=slices ']' !t_lookahead { ast.Subscript(value=a, slice=b, ctx=Store, LOCATIONS) }
    | star_atom

star_atom:
    | a=NAME { ast.Name(id=a.string, ctx=Store, LOCATIONS) }
    | '(' a=target_with_star_atom ')' { self.set_expr_context(a, Store) }
    | '(' a=[star_targets_tuple_seq] ')' { ast.Tuple(elts=a, ctx=Store, LOCATIONS) }
    | '[' a=[star_targets_list_seq] ']' {  ast.List(elts=a, ctx=Store, LOCATIONS) }

single_target:
    | single_subscript_attribute_target
    | a=NAME { ast.Name(id=a.string, ctx=Store, LOCATIONS) }
    | '(' a=single_target ')' { a }

single_subscript_attribute_target:
    | a=t_primary '.' b=NAME !t_lookahead { ast.Attribute(value=a, attr=b.string, ctx=Store, LOCATIONS) }
    | a=t_primary '[' b=slices ']' !t_lookahead { ast.Subscript(value=a, slice=b, ctx=Store, LOCATIONS) }


t_primary:
    | a=t_primary '.' b=NAME &t_lookahead { ast.Attribute(value=a, attr=b.string, ctx=Load, LOCATIONS) }
    | a=t_primary '[' b=slices ']' &t_lookahead { ast.Subscript(value=a, slice=b, ctx=Load, LOCATIONS) }
    | a=t_primary b=genexp &t_lookahead { ast.Call(func=a, args=[b], keywords=[], LOCATIONS) }
    | a=t_primary '(' b=[arguments] ')' &t_lookahead {
        ast.Call(
            func=a,
            args=b[0] if b else [],
            keywords=b[1] if b else [],
            LOCATIONS,
        )
     }
    | a=atom &t_lookahead { a }

t_lookahead: '(' | '[' | '.'

# Targets for del statements
# --------------------------

del_targets: a=','.del_target+ [','] { a }

del_target (memo):
    | a=t_primary '.' b=NAME !t_lookahead { ast.Attribute(value=a, attr=b.string, ctx=Del, LOCATIONS) }
    | a=t_primary '[' b=slices ']' !t_lookahead { ast.Subscript(value=a, slice=b, ctx=Del, LOCATIONS) }
    | del_t_atom

del_t_atom:
    | a=NAME { ast.Name(id=a.string, ctx=Del, LOCATIONS) }
    | '(' a=del_target ')' { self.set_expr_context(a, Del) }
    | '(' a=[del_targets] ')' { ast.Tuple(elts=a, ctx=Del, LOCATIONS) }
    | '[' a=[del_targets] ']' { ast.List(elts=a, ctx=Del, LOCATIONS) }


# TYPING ELEMENTS
# ---------------

# type_expressions allow */** but ignore them
type_expressions[list]:
    | a=','.expression+ ',' '*' b=expression ',' '**' c=expression { a + [b, c] }
    | a=','.expression+ ',' '*' b=expression { a + [b] }
    | a=','.expression+ ',' '**' b=expression { a + [b] }
    | '*' a=expression ',' '**' b=expression { [a, b] }
    | '*' a=expression { [a] }
    | '**' a=expression { [a] }
    | a=','.expression+ {a}

func_type_comment:
    | NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t.string }  # Must be followed by indented block
    | invalid_double_type_comments
    | TYPE_COMMENT

# ========================= END OF THE GRAMMAR ===========================



# ========================= START OF INVALID RULES =======================

# From here on, there are rules for invalid syntax with specialised error messages
invalid_arguments[NoReturn]:
    | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ {
        self.raise_syntax_error_starting_from(
            "iterable argument unpacking follows keyword argument unpacking",
            a,
        )
     }
    | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] {
        self.raise_syntax_error_known_range(
            "Generator expression must be parenthesized",
            a,
            (b[-1].ifs[-1] if b[-1].ifs else b[-1].iter)
        )
     }
    | a=NAME b='=' expression for_if_clauses {
        self.raise_syntax_error_known_range(
            "invalid syntax. Maybe you meant '==' or ':=' instead of '='?", a, b
        )
     }
    | (args ',')? a=NAME b='=' &(',' | ')') {
        self.raise_syntax_error_known_range("expected argument value expression", a, b)
     }
    | a=args b=for_if_clauses {
        self.raise_syntax_error_known_range(
            "Generator expression must be parenthesized",
            a[0][-1],
            (b[-1].ifs[-1] if b[-1].ifs else b[-1].iter),
        ) if len(a[0]) > 1 else None
     }
    | args ',' a=expression b=for_if_clauses {
        self.raise_syntax_error_known_range(
            "Generator expression must be parenthesized",
            a,
            (b[-1].ifs[-1] if b[-1].ifs else b[-1].iter),
        )
     }
    | a=args ',' args {
        self.raise_syntax_error(
            "positional argument follows keyword argument unpacking"
            if a[1][-1].arg is None else
            "positional argument follows keyword argument",
        )
     }
invalid_kwarg[NoReturn]:
    | a=('True'|'False'|'None') b='=' {
        self.raise_syntax_error_known_range(f"cannot assign to {a.string}", a, b)
     }
    | a=NAME b='=' expression for_if_clauses {
        self.raise_syntax_error_known_range(
            "invalid syntax. Maybe you meant '==' or ':=' instead of '='?", a, b
        )
     }
    | !(NAME '=') a=expression b='=' {
        self.raise_syntax_error_known_range(
            "expression cannot contain assignment, perhaps you meant \"==\"?", a, b,
        )
     }
    | a='**' expression '=' b=expression {
        self.raise_syntax_error_known_range(
            "cannot assign to keyword argument unpacking", a, b
        )
     }

# IMPORTANT: Note that the "_without_invalid" suffix causes the rule to
# not call invalid rules under it
expression_without_invalid[ast.AST]:
    | a=disjunction 'if' b=disjunction 'else' c=expression {
        ast.IfExp(body=b, test=a, orelse=c, LOCATIONS)
     }
    | disjunction
    | lambdef
invalid_legacy_expression:
    | a=NAME !'(' b=star_expressions {
        self.raise_syntax_error_known_range(
            f"Missing parentheses in call to '{a.string}' . Did you mean {a.string}(...)?", a, b,
        ) if a.string in ("exec", "print") else
        None
     }
invalid_expression[NoReturn]:
    # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
    # Soft keywords need to also be ignored because they can be parsed as NAME NAME
    | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid {
        (
            self.raise_syntax_error_known_range("invalid syntax. Perhaps you forgot a comma?", a, b)
            if not isinstance(a, ast.Name) or a.id not in ("print", "exec")
            else None
        )
     }
    | a=disjunction 'if' b=disjunction !('else'|':') {
        self.raise_syntax_error_known_range("expected 'else' after 'if' expression", a, b)
     }
    | a='lambda' [lambda_params] b=':' &(FSTRING_MIDDLE | tstring_middle_token | fstring_replacement_field | tstring_replacement_field)  {
        self.raise_syntax_error_known_range(
            "f-string: lambda expressions are not allowed without parentheses", a, b
        )
     }
invalid_named_expression[NoReturn]:
    | a=expression ':=' expression {
        self.raise_syntax_error_known_location(
            f"cannot use assignment expressions with {self.get_expr_name(a)}", a
        )
     }
    # Use in_raw_rule
    | a=NAME '=' b=bitwise_or !('='|':=') {
        (
            None
            if self.in_recursive_rule else
            self.raise_syntax_error_known_range(
                "invalid syntax. Maybe you meant '==' or ':=' instead of '='?", a, b
            )
        )
     }
    | !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':=') {
        (
            None
            if self.in_recursive_rule else
            self.raise_syntax_error_known_location(
                f"cannot assign to {self.get_expr_name(a)} here. Maybe you meant '==' instead of '='?", a
            )
        )
     }

invalid_assignment[NoReturn]:
    | a=invalid_ann_assign_target ':' expression {
        self.raise_syntax_error_known_location(
            f"only single target (not {self.get_expr_name(a)}) can be annotated", a
        )
     }
    | a=star_named_expression ',' star_named_expressions* ':' expression {
        self.raise_syntax_error_known_location("only single target (not tuple) can be annotated", a) }
    | a=expression ':' expression {
        self.raise_syntax_error_known_location("illegal target for annotation", a) }
    | (star_targets '=')* a=star_expressions '=' {
        self.raise_syntax_error_invalid_target(Target.STAR_TARGETS, a)
     }
    | (star_targets '=')* a=yield_expr '=' {
        self.raise_syntax_error_known_location("assignment to yield expression not possible", a)
     }
    | a=star_expressions augassign (yield_expr | star_expressions) {
        self.raise_syntax_error_known_location(
            f"'{self.get_expr_name(a)}' is an illegal expression for augmented assignment", a
        )
     }
invalid_ann_assign_target[ast.AST]:
    | a=list { a }
    | a=tuple { a }
    | '(' a=invalid_ann_assign_target ')' { a }
invalid_del_stmt[NoReturn]:
    | 'del' a=star_expressions {
        self.raise_syntax_error_invalid_target(Target.DEL_TARGETS, a)
     }
invalid_block[NoReturn]:
    | NEWLINE !INDENT { self.raise_indentation_error("expected an indented block") }
invalid_comprehension[NoReturn]:
    | ('[' | '(' | '{') a=starred_expression for_if_clauses {
        self.raise_syntax_error_known_location("iterable unpacking cannot be used in comprehension", a)
        if self.py_version < (3, 15)
        else None
     }
    | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses {
        self.raise_syntax_error_known_range(
            "did you forget parentheses around the comprehension target?", a, b[-1]
        )
     }
    | ('[' | '{') a=star_named_expression b=',' for_if_clauses {
        self.raise_syntax_error_known_range(
            "did you forget parentheses around the comprehension target?", a, b
        )
     }
invalid_dict_comprehension[NoReturn]:
    | '{' a='**' bitwise_or for_if_clauses '}' {
        self.raise_syntax_error_known_location("dict unpacking cannot be used in dict comprehension", a)
        if self.py_version < (3, 15)
        else None
     }
invalid_parameters[NoReturn]:
    | a="/" ',' {
        self.raise_syntax_error_known_location("at least one argument must precede /", a)
     }
    | (slash_no_default | slash_with_default) param_maybe_default* a='/' {
        self.raise_syntax_error_known_location("/ may appear only once", a)
     }
    | slash_no_default? param_no_default* invalid_parameters_helper a=param_no_default {
        self.raise_syntax_error_known_location(
            "parameter without a default follows parameter with a default", a
        )
     }
    | param_no_default* a='(' param_no_default+ ','? b=')' {
        self.raise_syntax_error_known_range(
            "Function parameters cannot be parenthesized", a, b
        )
     }
    | (slash_no_default | slash_with_default)? param_maybe_default* '*' (',' | param_no_default) param_maybe_default* a='/' {
        self.raise_syntax_error_known_location("/ must be ahead of *", a)
     }
    | param_maybe_default+ '/' a='*' {
        self.raise_syntax_error_known_location("expected comma between / and *", a)
     }
invalid_default:
    | a='=' &(')'|',') {
        self.raise_syntax_error_known_location("expected default value expression", a)
     }
invalid_star_etc:
    | a='*' (')' | ',' (')' | '**')) {
        self.raise_syntax_error_known_location("named arguments must follow bare *", a)
     }
    | '*' ',' TYPE_COMMENT { self.raise_syntax_error("bare * has associated type comment") }
    | '*' param a='=' {
        self.raise_syntax_error_known_location("var-positional argument cannot have default value", a)
     }
    | '*' (param_no_default | ',') param_maybe_default* a='*' (param_no_default | ',') {
        self.raise_syntax_error_known_location("* argument may appear only once", a)
     }
invalid_kwds:
    | '**' param a='=' {
        self.raise_syntax_error_known_location("var-keyword argument cannot have default value", a)
     }
    | '**' param ',' a=param {
        self.raise_syntax_error_known_location("arguments cannot follow var-keyword argument", a)
     }
    | '**' param ',' a=('*'|'**'|'/') {
        self.raise_syntax_error_known_location("arguments cannot follow var-keyword argument", a)
     }
invalid_parameters_helper: # This is only there to avoid type errors
    | a=slash_with_default { [a] }
    | a=param_with_default+
invalid_lambda_parameters[NoReturn]:
    | a="/" ',' {
        self.raise_syntax_error_known_location("at least one argument must precede /", a)
     }
    | (lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* a='/' {
        self.raise_syntax_error_known_location("/ may appear only once", a)
     }
    | lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default {
        self.raise_syntax_error_known_location(
            "parameter without a default follows parameter with a default", a
        )
     }
    | lambda_param_no_default* a='(' ','.lambda_param+ ','? b=')' {
        self.raise_syntax_error_known_range(
            "Lambda expression parameters cannot be parenthesized", a, b
        )
     }
    | (lambda_slash_no_default | lambda_slash_with_default)? lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* a='/' {
        self.raise_syntax_error_known_location("/ must be ahead of *", a)
     }
    | lambda_param_maybe_default+ '/' a='*' {
        self.raise_syntax_error_known_location("expected comma between / and *", a)
     }
invalid_lambda_parameters_helper[NoReturn]:
    | a=lambda_slash_with_default { [a] }
    | a=lambda_param_with_default+
invalid_lambda_star_etc[NoReturn]:
    | '*' (':' | ',' (':' | '**')) {
        self.raise_syntax_error("named arguments must follow bare *")
     }
    | '*' lambda_param a='=' {
        self.raise_syntax_error_known_location("var-positional argument cannot have default value", a)
     }
    | '*' (lambda_param_no_default | ',') lambda_param_maybe_default* a='*' (lambda_param_no_default | ',') {
        self.raise_syntax_error_known_location("* argument may appear only once", a)
     }
invalid_lambda_kwds:
    | '**' lambda_param a='=' {
        self.raise_syntax_error_known_location("var-keyword argument cannot have default value", a)
     }
    | '**' lambda_param ',' a=lambda_param {
        self.raise_syntax_error_known_location("arguments cannot follow var-keyword argument", a)
     }
    | '**' lambda_param ',' a=('*'|'**'|'/') {
        self.raise_syntax_error_known_location("arguments cannot follow var-keyword argument", a)
     }
invalid_double_type_comments[NoReturn]:
    | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT {
        self.raise_syntax_error("Cannot have two type comments on def")
     }
invalid_with_item[NoReturn]:
    | expression 'as' a=expression &(',' | ')' | ':') {
        self.raise_syntax_error_invalid_target(Target.STAR_TARGETS, a)
     }

invalid_for_target[NoReturn]:
    | 'async'? 'for' a=star_expressions {
        self.raise_syntax_error_invalid_target(Target.FOR_TARGETS, a)
     }

invalid_group[NoReturn]:
    | '(' a=starred_expression ')' {
        self.raise_syntax_error_known_location("cannot use starred expression here", a)
     }
    | '(' a='**' expression ')' {
        self.raise_syntax_error_known_location("cannot use double starred expression here", a)
     }
invalid_import:
    | a='import' ','.dotted_name+ 'from' dotted_name {
        self.raise_syntax_error_starting_from(
            "Did you mean to use 'from ... import ...' instead?", a
        )
     }
invalid_import_from:
    | 'from' ('.' | '...')* dotted_name a="lazy" 'import' import_from_targets {
        self.raise_syntax_error_known_location(
            "use 'lazy from ... ' instead of 'from ... lazy import'", a
        )
     }
invalid_import_from_targets[NoReturn]:
    | import_from_as_names ',' NEWLINE {
        self.raise_syntax_error("trailing comma not allowed without surrounding parentheses")
     }

invalid_with_stmt[None]:
    | ['async'] 'with' ','.(expression ['as' star_target])+ &&':' { UNREACHABLE }
    | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' &&':' { UNREACHABLE }
invalid_with_stmt_indent[NoReturn]:
    | ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'with' statement on line {a.start[0]}"
        )
     }
    | ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'with' statement on line {a.start[0]}"
        )
     }

invalid_try_stmt[NoReturn]:
    | a='try' ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'try' statement on line {a.start[0]}",
        )
     }
    | 'try' ':' block !('except' | 'finally') {
        self.raise_syntax_error("expected 'except' or 'finally' block")
     }
    | 'try' ':' block* except_block+ a='except' b='*' expression ['as' NAME] ':' {
        self.raise_syntax_error_known_range(
            "cannot have both 'except' and 'except*' on the same 'try'", a, b
        )
     }
    | 'try' ':' block* except_star_block+ a='except' [expression ['as' NAME]] ':' {
        self.raise_syntax_error_known_location(
            "cannot have both 'except' and 'except*' on the same 'try'", a
        )
     }
invalid_except_stmt[None]:
    | 'except' '*'? a=expression ',' expressions 'as' t=NAME ':' {
        self.raise_syntax_error_known_range(
            "multiple exception types must be parenthesized when using 'as'"
            if self.py_version >= (3, 14)
            else "multiple exception types must be parenthesized",
            a,
            t,
        )
     }
    | 'except' '*'? a=expression ',' b=expressions ':' {
        self.raise_syntax_error_known_range(
            "multiple exception types must be parenthesized",
            a,
            b,
        )
        if self.py_version < (3, 14)
        else None
     }
    | 'except' '*'? expression a=',' {
        self.raise_syntax_error_known_location("invalid syntax", a)
        if self.py_version < (3, 14)
        else None
     }
    | a='except' '*'? expression ['as' NAME ] NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='except' '*'? NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='except' '*' (NEWLINE | ':') {
        self.raise_syntax_error("expected one or more exception types")
     }
invalid_finally_stmt[NoReturn]:
    | a='finally' ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'finally' statement on line {a.start[0]}"
        )
     }
invalid_except_stmt_indent[NoReturn]:
    | a='except' expression ['as' NAME ] ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'except' statement on line {a.start[0]}"
        )
     }
    | a='except' ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'except' statement on line {a.start[0]}"
        )
     }
invalid_except_star_stmt_indent:
    | a='except' '*' expression ['as' NAME ] ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'except*' statement on line {a.start[0]}"
        )
     }
invalid_match_stmt[NoReturn]:
    | "match" subject_expr !':' {
        self.check_version(
            (3, 10),
            "Pattern matching is",
            self.raise_syntax_error("expected ':'")
        )
     }
    | a="match" subject=subject_expr ':' NEWLINE !INDENT {
        self.check_version(
            (3, 10),
            "Pattern matching is",
            self.raise_indentation_error(
                f"expected an indented block after 'match' statement on line {a.start[0]}"
            )
        )
     }
invalid_case_block[NoReturn]:
    | "case" patterns guard? !':' { self.raise_syntax_error("expected ':'") }
    | a="case" patterns guard? ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'case' statement on line {a.start[0]}"
        )
     }
invalid_as_pattern[NoReturn]:
    | or_pattern 'as' a="_" {
        self.raise_syntax_error_known_location("cannot use '_' as a target", a)
     }
    | or_pattern 'as' a=expression {
        self.raise_syntax_error_known_location(
            f"cannot use {self.get_expr_name(a)} as pattern target", a)
        if self.py_version >= (3, 14)
        else self.raise_syntax_error_known_location("invalid pattern target", a)
     }
invalid_class_pattern[NoReturn]:
    | name_or_attr '(' a=invalid_class_argument_pattern  {
        self.raise_syntax_error_known_range(
            "positional patterns follow keyword patterns", a[0], a[-1]
        )
     }
invalid_class_argument_pattern[list]:
    | [positional_patterns ','] keyword_patterns ',' a=positional_patterns { a }
invalid_if_stmt[NoReturn]:
    | 'if' named_expression NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='if' a=named_expression ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'if' statement on line {a.start[0]}"
        )
     }
invalid_elif_stmt[NoReturn]:
    | 'elif' named_expression NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='elif' named_expression ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'elif' statement on line {a.start[0]}"
        )
     }
invalid_else_stmt[NoReturn]:
    | a='else' ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'else' statement on line {a.start[0]}"
        )
     }
invalid_while_stmt[NoReturn]:
    | 'while' named_expression NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='while' named_expression ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'while' statement on line {a.start[0]}"
        )
     }
invalid_for_stmt[NoReturn]:
    | ['async'] 'for' star_targets 'in' star_expressions NEWLINE { self.raise_syntax_error("expected ':'") }
    | ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after 'for' statement on line {a.start[0]}"
        )
     }
invalid_def_raw[NoReturn]:
    | ['async'] a='def' NAME  [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after function definition on line {a.start[0]}"
        )
     }
invalid_class_def_raw[NoReturn]:
    | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { self.raise_syntax_error("expected ':'") }
    | a='class' NAME  [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT {
        self.raise_indentation_error(
            f"expected an indented block after class definition on line {a.start[0]}"
        )
     }

invalid_double_starred_kvpairs[None]:
    | ','.double_starred_kvpair+ ',' invalid_kvpair
    | expression ':' a='*' bitwise_or {
        self.raise_syntax_error_starting_from("cannot use a starred expression in a dictionary value", a)
     }
    | expression a=':' &('}'|',') {
        self.raise_syntax_error_known_location("expression expected after dictionary key and ':'", a)
     }
invalid_kvpair[None]:
    | a=expression !(':') {
        self.raise_raw_syntax_error(
            "':' expected after dictionary key",
            (a.lineno, a.col_offset),
            (a.end_lineno, a.end_col_offset)
        )
     }
    | expression ':' a='*' bitwise_or {
        self.raise_syntax_error_starting_from("cannot use a starred expression in a dictionary value", a)
     }
    | expression a=':' &('}'|',') {
        self.raise_syntax_error_known_location(
            "expression expected after dictionary key and ':'", a
        )
     }
    | expression a=':' {
        self.raise_syntax_error_known_location("expression expected after dictionary key and ':'", a)
     }
invalid_starred_expression:
    | a='*' expression '=' b=expression {
        self.raise_syntax_error_known_range(
            "cannot assign to iterable argument unpacking", a, b
        )
     }
invalid_replacement_field:
    | '{' a='=' { self.raise_syntax_error_known_location("f-string: valid expression required before '='", a) }
    | '{' a='!' { self.raise_syntax_error_known_location("f-string: valid expression required before '!'", a) }
    | '{' a=':' { self.raise_syntax_error_known_location("f-string: valid expression required before ':'", a) }
    | '{' a='}' { self.raise_syntax_error_known_location("f-string: valid expression required before '}'", a) }
    | '{' !(yield_expr | star_expressions) {
        self.raise_syntax_error_on_next_token(
            "f-string: expecting a valid expression after '{'"
        )
     }
    | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') {
        self.raise_syntax_error_on_next_token("f-string: expecting '=', or '!', or ':', or '}'") }
    | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') {
       self.raise_syntax_error_on_next_token("f-string: expecting '!', or ':', or '}'")
     }
    | '{' (yield_expr | star_expressions) '='? invalid_conversion_character
    | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') {
        self.raise_syntax_error_on_next_token("f-string: expecting ':' or '}'")
     }
    | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' {
        self.raise_syntax_error_on_next_token("f-string: expecting '}', or format specs")
     }
    | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' {
        self.raise_syntax_error_on_next_token("f-string: expecting '}'")
     }

invalid_conversion_character:
    | '!' &(':' | '}') { self.raise_syntax_error_on_next_token("f-string: missing conversion character") }
    | '!' !NAME { self.raise_syntax_error_on_next_token("f-string: invalid conversion character") }