From e446575f46d4a9f6d5249b8eb1922d7566851bfb Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Thu, 15 Oct 2026 17:12:27 +0000 Subject: [PATCH 1/3] fix(gazelle): migrate to official tree-sitter Go bindings Replace the unmaintained `github.com/smacker/go-tree-sitter` dependency with the official `github.com/tree-sitter/go-tree-sitter` and `github.com/tree-sitter/tree-sitter-python` (v0.25.0) modules. This updates the Python grammar so the Gazelle plugin can parse Python 3.14 PEP 758 unparenthesized multi-exception `except` clauses without errors. Because the official bindings include C sources across parent and subdirectory paths that standard `go_repository` Gazelle generation cannot compile directly, fetch both modules via `http_archive` with custom `cc_library` and `go_library` targets in both Bzlmod and WORKSPACE modes. Generated by Gemini. Tested on our large-scale QuantumAI monorepo. Fixes 3416. I hope. --- gazelle/BUILD.bazel | 4 +- gazelle/MODULE.bazel | 19 ++- gazelle/deps.bzl | 113 ++++++++++++++++-- gazelle/go.mod | 4 +- gazelle/go.sum | 8 +- gazelle/internal/go_tree_sitter_BUILD.bazel | 47 ++++++++ .../internal/tree_sitter_python_BUILD.bazel | 36 ++++++ gazelle/manifest/BUILD.bazel | 4 +- gazelle/python/BUILD.bazel | 10 +- gazelle/python/file_parser.go | 101 +++++++++------- gazelle/python/file_parser_test.go | 35 ++++++ gazelle/pythonconfig/BUILD.bazel | 2 +- news/3416.changed.md | 4 + 13 files changed, 322 insertions(+), 65 deletions(-) create mode 100644 gazelle/internal/go_tree_sitter_BUILD.bazel create mode 100644 gazelle/internal/tree_sitter_python_BUILD.bazel create mode 100644 news/3416.changed.md diff --git a/gazelle/BUILD.bazel b/gazelle/BUILD.bazel index 30758181ed..7467648c6b 100644 --- a/gazelle/BUILD.bazel +++ b/gazelle/BUILD.bazel @@ -6,7 +6,9 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") # gazelle:prefix github.com/bazel-contrib/rules_python/gazelle # gazelle:exclude bazel-out # gazelle:exclude deps.bzl +# gazelle:exclude internal # gazelle:exclude internal_dev_deps.bzl +# gazelle:resolve go github.com/tree-sitter/tree-sitter-python/bindings/go @com_github_tree_sitter_tree_sitter_python//:go_default_library gazelle( name = "gazelle", ) @@ -36,7 +38,7 @@ filegroup( "//modules_mapping:distribution", "//python:distribution", "//pythonconfig:distribution", - ], + ] + glob(["internal/**"]), visibility = ["@rules_python//:__pkg__"], ) diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index b6eed23c6f..407be5338f 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -23,12 +23,29 @@ use_repo( "com_github_bmatcuk_doublestar_v4", "com_github_emirpasic_gods", "com_github_ghodss_yaml", - "com_github_smacker_go_tree_sitter", + "com_github_mattn_go_pointer", "com_github_stretchr_testify", "in_gopkg_yaml_v2", "org_golang_x_sync", ) +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "com_github_tree_sitter_go_tree_sitter", + build_file = "//:internal/go_tree_sitter_BUILD.bazel", + integrity = "sha256-rAsyh/Kydqt24TBaA6H14obQlkaQWwj1QNec3xza6NM=", + strip_prefix = "go-tree-sitter-c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5", + url = "https://github.com/tree-sitter/go-tree-sitter/archive/c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5.zip", +) + +http_archive( + name = "com_github_tree_sitter_tree_sitter_python", + build_file = "//:internal/tree_sitter_python_BUILD.bazel", + integrity = "sha256-e86IfrLzPpS/dKaWRc9RONQJZyDlT9MmmmEkwGuTxYQ=", + url = "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.25.0/tree-sitter-python.tar.gz", +) + python_stdlib_list = use_extension("//python:extensions.bzl", "python_stdlib_list") use_repo( python_stdlib_list, diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index c57d5193b8..83fd230416 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -31,7 +31,26 @@ def python_stdlib_list_deps(): url = "https://files.pythonhosted.org/packages/8c/25/f1540879c8815387980e56f973e54605bd924612399ace31487f7444171c/stdlib_list-0.12.0.tar.gz", ) +def tree_sitter_deps(): + "Fetch tree-sitter dependencies with custom BUILD files" + if "com_github_tree_sitter_go_tree_sitter" not in native.existing_rules(): + http_archive( + name = "com_github_tree_sitter_go_tree_sitter", + build_file = Label("//:internal/go_tree_sitter_BUILD.bazel"), + integrity = "sha256-rAsyh/Kydqt24TBaA6H14obQlkaQWwj1QNec3xza6NM=", + strip_prefix = "go-tree-sitter-c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5", + url = "https://github.com/tree-sitter/go-tree-sitter/archive/c9492002f76ed75037e3fe6d3bbabb54ed3e1ff5.zip", + ) + if "com_github_tree_sitter_tree_sitter_python" not in native.existing_rules(): + http_archive( + name = "com_github_tree_sitter_tree_sitter_python", + build_file = Label("//:internal/tree_sitter_python_BUILD.bazel"), + integrity = "sha256-e86IfrLzPpS/dKaWRc9RONQJZyDlT9MmmmEkwGuTxYQ=", + url = "https://github.com/tree-sitter/tree-sitter-python/releases/download/v0.25.0/tree-sitter-python.tar.gz", + ) + def gazelle_deps(): + tree_sitter_deps() go_deps() python_stdlib_list_deps() @@ -110,18 +129,18 @@ def go_deps(): sum = "h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=", version = "v0.6.0", ) + go_repository( + name = "com_github_mattn_go_pointer", + importpath = "github.com/mattn/go-pointer", + sum = "h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0=", + version = "v0.0.1", + ) go_repository( name = "com_github_pmezard_go_difflib", importpath = "github.com/pmezard/go-difflib", sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=", version = "v1.0.0", ) - go_repository( - name = "com_github_smacker_go_tree_sitter", - importpath = "github.com/smacker/go-tree-sitter", - sum = "h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=", - version = "v0.0.0-20240827094217-dd81d9e9be82", - ) go_repository( name = "com_github_stretchr_objx", importpath = "github.com/stretchr/objx", @@ -131,8 +150,86 @@ def go_deps(): go_repository( name = "com_github_stretchr_testify", importpath = "github.com/stretchr/testify", - sum = "h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=", - version = "v1.9.0", + sum = "h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=", + version = "v1.10.0", + ) + go_repository( + name = "com_github_tree_sitter_go_tree_sitter", + importpath = "github.com/tree-sitter/go-tree-sitter", + sum = "h1:B9WMBx89/TkqtmwfP+7OhREoGB6RdgmGk9ErlbiTOFI=", + version = "v0.24.1-0.20251112183152-c9492002f76e", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_c", + importpath = "github.com/tree-sitter/tree-sitter-c", + sum = "h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c=", + version = "v0.23.4", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_cpp", + importpath = "github.com/tree-sitter/tree-sitter-cpp", + sum = "h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w=", + version = "v0.23.4", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_embedded_template", + importpath = "github.com/tree-sitter/tree-sitter-embedded-template", + sum = "h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg=", + version = "v0.23.2", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_go", + importpath = "github.com/tree-sitter/tree-sitter-go", + sum = "h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA=", + version = "v0.23.4", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_html", + importpath = "github.com/tree-sitter/tree-sitter-html", + sum = "h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc=", + version = "v0.23.2", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_java", + importpath = "github.com/tree-sitter/tree-sitter-java", + sum = "h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig=", + version = "v0.23.5", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_javascript", + importpath = "github.com/tree-sitter/tree-sitter-javascript", + sum = "h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo=", + version = "v0.23.1", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_json", + importpath = "github.com/tree-sitter/tree-sitter-json", + sum = "h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ=", + version = "v0.24.8", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_php", + importpath = "github.com/tree-sitter/tree-sitter-php", + sum = "h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8=", + version = "v0.23.11", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_python", + importpath = "github.com/tree-sitter/tree-sitter-python", + sum = "h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0=", + version = "v0.25.0", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_ruby", + importpath = "github.com/tree-sitter/tree-sitter-ruby", + sum = "h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo=", + version = "v0.23.1", + ) + go_repository( + name = "com_github_tree_sitter_tree_sitter_rust", + importpath = "github.com/tree-sitter/tree-sitter-rust", + sum = "h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90=", + version = "v0.23.2", ) go_repository( name = "in_gopkg_check_v1", diff --git a/gazelle/go.mod b/gazelle/go.mod index 9ad4951536..cdf28f264f 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -11,8 +11,9 @@ require ( github.com/bmatcuk/doublestar/v4 v4.9.1 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 - github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 github.com/stretchr/testify v1.9.0 + github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e + github.com/tree-sitter/tree-sitter-python v0.25.0 golang.org/x/sync v0.11.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -20,6 +21,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/mod v0.23.0 // indirect golang.org/x/sys v0.30.0 // indirect diff --git a/gazelle/go.sum b/gazelle/go.sum index d5fc2b1af9..1b141b6642 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum @@ -14,12 +14,16 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4= -github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e h1:B9WMBx89/TkqtmwfP+7OhREoGB6RdgmGk9ErlbiTOFI= +github.com/tree-sitter/go-tree-sitter v0.24.1-0.20251112183152-c9492002f76e/go.mod h1:r77ig7BikoZhHrrsjAnv8RqGti5rtSyvDHPzgTPsUuU= +github.com/tree-sitter/tree-sitter-python v0.25.0 h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0= +github.com/tree-sitter/tree-sitter-python v0.25.0/go.mod h1:cpdthSy/Yoa28aJFBscFHlGiU+cnSiSh1kuDVtI8YeM= golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= diff --git a/gazelle/internal/go_tree_sitter_BUILD.bazel b/gazelle/internal/go_tree_sitter_BUILD.bazel new file mode 100644 index 0000000000..901e614cf0 --- /dev/null +++ b/gazelle/internal/go_tree_sitter_BUILD.bazel @@ -0,0 +1,47 @@ +"""Build file for github.com/tree-sitter/go-tree-sitter.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "tree_sitter_sources", + hdrs = glob([ + "include/**/*.h", + "src/**/*.h", + "src/**/*.c", + ]), + defines = [ + "_DEFAULT_SOURCE", + "_POSIX_C_SOURCE=200112L", + ], + includes = [ + "include", + "src", + ], + visibility = ["//visibility:private"], +) + +go_library( + name = "go-tree-sitter", + srcs = glob( + ["*.go"], + exclude = ["*_test.go"], + ) + [ + "allocator.c", + "allocator.h", + ], + cdeps = [":tree_sitter_sources"], + cgo = True, + copts = [ + "-std=c11", + ], + importpath = "github.com/tree-sitter/go-tree-sitter", + visibility = ["//visibility:public"], + deps = ["@com_github_mattn_go_pointer//:go_default_library"], +) + +alias( + name = "go_default_library", + actual = ":go-tree-sitter", + visibility = ["//visibility:public"], +) diff --git a/gazelle/internal/tree_sitter_python_BUILD.bazel b/gazelle/internal/tree_sitter_python_BUILD.bazel new file mode 100644 index 0000000000..1f6fe1fa81 --- /dev/null +++ b/gazelle/internal/tree_sitter_python_BUILD.bazel @@ -0,0 +1,36 @@ +"""Build file for github.com/tree-sitter/tree-sitter-python.""" + +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "tree_sitter_python_sources", + hdrs = [ + "src/parser.c", + "src/scanner.c", + "src/tree_sitter/alloc.h", + "src/tree_sitter/array.h", + "src/tree_sitter/parser.h", + ], + includes = ["bindings/go"], + visibility = ["//visibility:private"], +) + +go_library( + name = "go", + srcs = ["bindings/go/binding.go"], + cdeps = [":tree_sitter_python_sources"], + cgo = True, + copts = [ + "-std=c11", + "-fPIC", + ], + importpath = "github.com/tree-sitter/tree-sitter-python/bindings/go", + visibility = ["//visibility:public"], +) + +alias( + name = "go_default_library", + actual = ":go", + visibility = ["//visibility:public"], +) diff --git a/gazelle/manifest/BUILD.bazel b/gazelle/manifest/BUILD.bazel index 68f4e1be30..fc12c5e435 100644 --- a/gazelle/manifest/BUILD.bazel +++ b/gazelle/manifest/BUILD.bazel @@ -12,8 +12,8 @@ go_library( importpath = "github.com/bazel-contrib/rules_python/gazelle/manifest", visibility = ["//visibility:public"], deps = [ - "@com_github_emirpasic_gods//sets/treeset:go_default_library", - "@in_gopkg_yaml_v2//:go_default_library", + "@com_github_emirpasic_gods//sets/treeset", + "@in_gopkg_yaml_v2//:yaml_v2", ], ) diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 1ffa2890e1..0b9fd761b5 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -38,11 +38,11 @@ go_library( "@bazel_gazelle//rule", "@com_github_bazelbuild_buildtools//build", "@com_github_bmatcuk_doublestar_v4//:doublestar", - "@com_github_emirpasic_gods//lists/singlylinkedlist:go_default_library", - "@com_github_emirpasic_gods//sets/treeset:go_default_library", - "@com_github_emirpasic_gods//utils:go_default_library", - "@com_github_smacker_go_tree_sitter//:go_default_library", - "@com_github_smacker_go_tree_sitter//python:go_default_library", + "@com_github_emirpasic_gods//lists/singlylinkedlist", + "@com_github_emirpasic_gods//sets/treeset", + "@com_github_emirpasic_gods//utils", + "@com_github_tree_sitter_go_tree_sitter//:go-tree-sitter", + "@com_github_tree_sitter_tree_sitter_python//:go_default_library", "@org_golang_x_sync//errgroup", ], ) diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index e129337e11..97fdf551eb 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -22,8 +22,8 @@ import ( "path/filepath" "strings" - sitter "github.com/smacker/go-tree-sitter" - "github.com/smacker/go-tree-sitter/python" + sitter "github.com/tree-sitter/go-tree-sitter" + tree_sitter_python "github.com/tree-sitter/tree-sitter-python/bindings/go" ) const ( @@ -47,31 +47,31 @@ type ParserOutput struct { } type FileParser struct { - code []byte - relFilepath string - output ParserOutput - inTypeCheckingBlock bool + code []byte + relFilepath string + output ParserOutput + inTypeCheckingBlock bool } func NewFileParser() *FileParser { return &FileParser{} } -// ParseCode instantiates a new tree-sitter Parser and parses the python code, returning -// the tree-sitter RootNode. -// It prints a warning if parsing fails. -func ParseCode(code []byte, path string) (*sitter.Node, error) { +func parseCodeToTree(code []byte, path string) (*sitter.Tree, error) { parser := sitter.NewParser() - parser.SetLanguage(python.GetLanguage()) - - tree, err := parser.ParseCtx(context.Background(), nil, code) - if err != nil { + defer parser.Close() + if err := parser.SetLanguage(sitter.NewLanguage(tree_sitter_python.Language())); err != nil { return nil, err } + tree := parser.ParseCtx(context.Background(), code, nil) + if tree == nil { + return nil, fmt.Errorf("failed to parse %q", path) + } + root := tree.RootNode() if !root.HasError() { - return root, nil + return tree, nil } log.Printf("WARNING: failed to parse %q. The resulting BUILD target may be incorrect.", path) @@ -80,43 +80,54 @@ func ParseCode(code []byte, path string) (*sitter.Node, error) { // failure may be in some part of the code that Gazelle doesn't care about. verbose, envExists := os.LookupEnv("RULES_PYTHON_GAZELLE_VERBOSE") if !envExists || verbose != "1" { - return root, nil + return tree, nil } - for i := 0; i < int(root.ChildCount()); i++ { + for i := uint(0); i < root.ChildCount(); i++ { child := root.Child(i) if child.IsError() { // Example logs: // gazelle: Parse error at {Row:1 Column:0}: // def search_one_more_level[T](): - log.Printf("Parse error at %+v:\n%+v", child.StartPoint(), child.Content(code)) + log.Printf("Parse error at %+v:\n%+v", child.StartPosition(), child.Utf8Text(code)) // Log the internal tree-sitter representation of what was parsed. Eg: // gazelle: The above was parsed as: (ERROR (identifier) (call function: (list (identifier)) arguments: (argument_list))) - log.Printf("The above was parsed as: %v", child.String()) + log.Printf("The above was parsed as: %v", child.ToSexp()) } } - return root, nil + return tree, nil +} + +// ParseCode instantiates a new tree-sitter Parser and parses the python code, returning +// the tree-sitter RootNode. +// It prints a warning if parsing fails. +func ParseCode(code []byte, path string) (*sitter.Node, error) { + tree, err := parseCodeToTree(code, path) + if err != nil { + return nil, err + } + return tree.RootNode(), nil } // parseMain returns true if the python file has an `if __name__ == "__main__":` block, // which is a common idiom for python scripts/binaries. func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool { - for i := 0; i < int(node.ChildCount()); i++ { + for i := uint(0); i < node.ChildCount(); i++ { if err := ctx.Err(); err != nil { return false } child := node.Child(i) - if child.Type() == sitterNodeTypeIfStatement && - child.Child(1).Type() == sitterNodeTypeComparisonOperator && child.Child(1).Child(1).Type() == "==" { + if child.Kind() == sitterNodeTypeIfStatement && + child.Child(1).Kind() == sitterNodeTypeComparisonOperator && child.Child(1).Child(1).Kind() == "==" { statement := child.Child(1) a, b := statement.Child(0), statement.Child(2) // convert "'__main__' == __name__" to "__name__ == '__main__'" - if b.Type() == sitterNodeTypeIdentifier { + if b.Kind() == sitterNodeTypeIdentifier { a, b = b, a } - if a.Type() == sitterNodeTypeIdentifier && a.Content(p.code) == "__name__" && - b.Type() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" { + if a.Kind() == sitterNodeTypeIdentifier && a.Utf8Text(p.code) == "__name__" && + b.Kind() == sitterNodeTypeString && string(p.code[b.StartByte()+1:b.EndByte()-1]) == "__main__" { return true } } @@ -127,18 +138,18 @@ func (p *FileParser) parseMain(ctx context.Context, node *sitter.Node) bool { // parseImportStatement parses a node for an import statement, returning a `Module` and a boolean // representing if the parse was OK or not. func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { - switch node.Type() { + switch node.Kind() { case sitterNodeTypeDottedName: return Module{ - Name: node.Content(code), - LineNumber: node.StartPoint().Row + 1, + Name: node.Utf8Text(code), + LineNumber: uint32(node.StartPosition().Row) + 1, }, true case sitterNodeTypeAliasedImport: return parseImportStatement(node.Child(0), code) case sitterNodeTypeWildcardImport: return Module{ Name: "*", - LineNumber: node.StartPoint().Row + 1, + LineNumber: uint32(node.StartPosition().Row) + 1, }, true } return Module{}, false @@ -158,8 +169,8 @@ func cleanImportString(s string) string { // an import statement. It updates FileParser.output.Modules with the `module` that the // import represents. func (p *FileParser) parseImportStatements(node *sitter.Node) bool { - if node.Type() == sitterNodeTypeImportStatement { - for j := 1; j < int(node.ChildCount()); j++ { + if node.Kind() == sitterNodeTypeImportStatement { + for j := uint(1); j < node.ChildCount(); j++ { m, ok := parseImportStatement(node.Child(j), p.code) if !ok { continue @@ -173,15 +184,15 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } p.output.Modules = append(p.output.Modules, m) } - } else if node.Type() == sitterNodeTypeImportFromStatement { - from := node.Child(1).Content(p.code) + } else if node.Kind() == sitterNodeTypeImportFromStatement { + from := node.Child(1).Utf8Text(p.code) from = cleanImportString(from) // If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1. // If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules. if from == "." { return true } - for j := 3; j < int(node.ChildCount()); j++ { + for j := uint(3); j < node.ChildCount(); j++ { m, ok := parseImportStatement(node.Child(j), p.code) if !ok { continue @@ -202,8 +213,8 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { // parseComments parses a node for comments, returning true if the node is a comment. // It updates FileParser.output.Comments with the parsed comment. func (p *FileParser) parseComments(node *sitter.Node) bool { - if node.Type() == sitterNodeTypeComment { - p.output.Comments = append(p.output.Comments, Comment(node.Content(p.code))) + if node.Kind() == sitterNodeTypeComment { + p.output.Comments = append(p.output.Comments, Comment(node.Utf8Text(p.code))) return true } return false @@ -217,23 +228,23 @@ func (p *FileParser) SetCodeAndFile(code []byte, relPackagePath, filename string // isTypeCheckingBlock returns true if the given node is an `if TYPE_CHECKING:` block. func (p *FileParser) isTypeCheckingBlock(node *sitter.Node) bool { - if node.Type() != sitterNodeTypeIfStatement || node.ChildCount() < 2 { + if node.Kind() != sitterNodeTypeIfStatement || node.ChildCount() < 2 { return false } condition := node.Child(1) // Handle `if TYPE_CHECKING:` - if condition.Type() == sitterNodeTypeIdentifier && condition.Content(p.code) == "TYPE_CHECKING" { + if condition.Kind() == sitterNodeTypeIdentifier && condition.Utf8Text(p.code) == "TYPE_CHECKING" { return true } // Handle `if typing.TYPE_CHECKING:` - if condition.Type() == "attribute" && condition.ChildCount() >= 3 { + if condition.Kind() == "attribute" && condition.ChildCount() >= 3 { object := condition.Child(0) attr := condition.Child(2) - if object.Type() == sitterNodeTypeIdentifier && object.Content(p.code) == "typing" && - attr.Type() == sitterNodeTypeIdentifier && attr.Content(p.code) == "TYPE_CHECKING" { + if object.Kind() == sitterNodeTypeIdentifier && object.Utf8Text(p.code) == "typing" && + attr.Kind() == sitterNodeTypeIdentifier && attr.Utf8Text(p.code) == "TYPE_CHECKING" { return true } } @@ -252,7 +263,7 @@ func (p *FileParser) parse(ctx context.Context, node *sitter.Node) { p.inTypeCheckingBlock = true } - for i := 0; i < int(node.ChildCount()); i++ { + for i := uint(0); i < node.ChildCount(); i++ { if err := ctx.Err(); err != nil { return } @@ -271,10 +282,12 @@ func (p *FileParser) parse(ctx context.Context, node *sitter.Node) { } func (p *FileParser) Parse(ctx context.Context) (*ParserOutput, error) { - rootNode, err := ParseCode(p.code, p.relFilepath) + tree, err := parseCodeToTree(p.code, p.relFilepath) if err != nil { return nil, err } + defer tree.Close() + rootNode := tree.RootNode() p.output.HasMain = p.parseMain(ctx, rootNode) diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index 0a6fd1b4ab..81c5b99452 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go @@ -383,3 +383,38 @@ from foo\ assert.ElementsMatch(t, expected, output.Modules) }) } + +func TestParsePEP758UnparenthesizedExcept(t *testing.T) { + t.Parallel() + code := []byte(`import os + +try: + pass +except ValueError, OSError, OverflowError: + import sys + +if __name__ == "__main__": + pass +`) + rootNode, err := ParseCode(code, "pep758.py") + assert.NoError(t, err) + assert.False(t, rootNode.HasError(), "expected PEP 758 except clause to parse without error") + + p := NewFileParser() + p.SetCodeAndFile(code, "", "pep758.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + assert.True(t, output.HasMain) + assert.Equal(t, []Module{ + { + Name: "os", + LineNumber: 1, + Filepath: "pep758.py", + }, + { + Name: "sys", + LineNumber: 6, + Filepath: "pep758.py", + }, + }, output.Modules) +} diff --git a/gazelle/pythonconfig/BUILD.bazel b/gazelle/pythonconfig/BUILD.bazel index a82a7f19ca..9c1e5eb9a7 100644 --- a/gazelle/pythonconfig/BUILD.bazel +++ b/gazelle/pythonconfig/BUILD.bazel @@ -11,7 +11,7 @@ go_library( deps = [ "//manifest", "@bazel_gazelle//label", - "@com_github_emirpasic_gods//lists/singlylinkedlist:go_default_library", + "@com_github_emirpasic_gods//lists/singlylinkedlist", ], ) diff --git a/news/3416.changed.md b/news/3416.changed.md new file mode 100644 index 0000000000..5aab21bfd1 --- /dev/null +++ b/news/3416.changed.md @@ -0,0 +1,4 @@ +(gazelle) Migrated the Python AST parser from `smacker/go-tree-sitter` to the +official `tree-sitter/go-tree-sitter` and `tree-sitter/tree-sitter-python` +packages, adding support for Python 3.14 PEP 758 unparenthesized exception +clauses ([#3416](https://github.com/bazel-contrib/rules_python/issues/3416)). From e28d515c6bbda485d956588efe96b3480f42be2e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Thu, 15 Oct 2026 17:58:52 +0000 Subject: [PATCH 2/3] add docstring --- gazelle/python/file_parser.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index 97fdf551eb..7bc3838cc4 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -57,6 +57,10 @@ func NewFileParser() *FileParser { return &FileParser{} } +// parseCodeToTree instantiates a new tree-sitter Parser and parses the Python code, +// returning the parsed tree-sitter Tree. Callers are responsible for calling Close() +// on the returned Tree to release its underlying C resources when finished. +// It logs a warning if the syntax tree contains parse errors. func parseCodeToTree(code []byte, path string) (*sitter.Tree, error) { parser := sitter.NewParser() defer parser.Close() From bc5b12a5e160e7f91f665fc730b9b305dc3471b4 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Thu, 15 Oct 2026 17:59:25 +0000 Subject: [PATCH 3/3] Remove unneeded tree_sitter grammars --- gazelle/deps.bzl | 97 ++++++++++-------------------------------------- 1 file changed, 19 insertions(+), 78 deletions(-) diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index 83fd230416..a7868c666a 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -55,6 +55,25 @@ def gazelle_deps(): python_stdlib_list_deps() def go_deps(): + # Prevent `bazel run //:gazelle_update_repos` from generating `go_repository` + # entries for the repos managed by `tree_sitter_deps()` or for the 11 other + # language grammars required by upstream `go-tree-sitter/go.mod` for its own + # internal unit tests. Without these directives, `gazelle update-repos` sees + # those modules in `go list -m all` and re-adds them here; declaring + # `# gazelle:repository` directives tells `update-repos` to skip them. + # gazelle:repository http_archive name=com_github_tree_sitter_go_tree_sitter + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_c + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_cpp + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_embedded_template + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_go + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_html + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_java + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_javascript + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_json + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_php + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_python + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_ruby + # gazelle:repository http_archive name=com_github_tree_sitter_tree_sitter_rust "Fetch go dependencies" go_repository( name = "com_github_bazelbuild_bazel_gazelle", @@ -153,84 +172,6 @@ def go_deps(): sum = "h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=", version = "v1.10.0", ) - go_repository( - name = "com_github_tree_sitter_go_tree_sitter", - importpath = "github.com/tree-sitter/go-tree-sitter", - sum = "h1:B9WMBx89/TkqtmwfP+7OhREoGB6RdgmGk9ErlbiTOFI=", - version = "v0.24.1-0.20251112183152-c9492002f76e", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_c", - importpath = "github.com/tree-sitter/tree-sitter-c", - sum = "h1:nBPH3FV07DzAD7p0GfNvXM+Y7pNIoPenQWBpvM++t4c=", - version = "v0.23.4", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_cpp", - importpath = "github.com/tree-sitter/tree-sitter-cpp", - sum = "h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJTJFCyFIeZU/8w=", - version = "v0.23.4", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_embedded_template", - importpath = "github.com/tree-sitter/tree-sitter-embedded-template", - sum = "h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg=", - version = "v0.23.2", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_go", - importpath = "github.com/tree-sitter/tree-sitter-go", - sum = "h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA=", - version = "v0.23.4", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_html", - importpath = "github.com/tree-sitter/tree-sitter-html", - sum = "h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc=", - version = "v0.23.2", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_java", - importpath = "github.com/tree-sitter/tree-sitter-java", - sum = "h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig=", - version = "v0.23.5", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_javascript", - importpath = "github.com/tree-sitter/tree-sitter-javascript", - sum = "h1:1fWupaRC0ArlHJ/QJzsfQ3Ibyopw7ZfQK4xXc40Zveo=", - version = "v0.23.1", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_json", - importpath = "github.com/tree-sitter/tree-sitter-json", - sum = "h1:tV5rMkihgtiOe14a9LHfDY5kzTl5GNUYe6carZBn0fQ=", - version = "v0.24.8", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_php", - importpath = "github.com/tree-sitter/tree-sitter-php", - sum = "h1:iHewsLNDmznh8kgGyfWfujsZxIz1YGbSd2ZTEM0ZiP8=", - version = "v0.23.11", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_python", - importpath = "github.com/tree-sitter/tree-sitter-python", - sum = "h1:O6XD9v8U1LOcRc3cNj9nM7XufrtEBezE6VrpRrHZDf0=", - version = "v0.25.0", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_ruby", - importpath = "github.com/tree-sitter/tree-sitter-ruby", - sum = "h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV6z8Qw8ai+72bYo=", - version = "v0.23.1", - ) - go_repository( - name = "com_github_tree_sitter_tree_sitter_rust", - importpath = "github.com/tree-sitter/tree-sitter-rust", - sum = "h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90=", - version = "v0.23.2", - ) go_repository( name = "in_gopkg_check_v1", importpath = "gopkg.in/check.v1",