diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index 8f91a03b4c..c77167b593 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -17,8 +17,10 @@ EXTRACTED_WHEEL_FILES = "extracted_whl_files" WHEEL_FILE_PUBLIC_LABEL = "whl" WHEEL_FILE_IMPL_LABEL = "_whl" +WHEEL_FILE = "whl_file" PY_LIBRARY_PUBLIC_LABEL = "pkg" PY_LIBRARY_IMPL_LABEL = "_pkg" +PY_SRCS_LABEL = "srcs" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index b66e502681..9a150db9e8 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -682,7 +682,6 @@ whl_archive = repository_rule( "sha256", "urls", "whl_patches", - # common attrs "enable_implicit_namespace_pkgs", "envsubst", "pip_data_exclude", diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 933b529053..217210e782 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -26,6 +26,8 @@ load( "EXTRACTED_WHEEL_FILES", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", + "PY_SRCS_LABEL", + "WHEEL_FILE", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) @@ -57,6 +59,19 @@ def whl_library_targets_from_requires( entry_points = {}, include = [], group_deps = [], + group_name = None, + dep_template = None, + data_exclude = [], + enable_implicit_namespace_pkgs = False, + sdist_filename = None, + namespace_package_files = [], + filegroups = None, + copy_files = {}, + copy_executables = {}, + srcs_exclude = [], + data = [], + visibility = ["//visibility:public"], + tags = [], **kwargs): """The macro to create whl targets from the METADATA. @@ -73,8 +88,44 @@ def whl_library_targets_from_requires( extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. - **kwargs: Extra args passed to the {obj}`whl_library_targets` + group_name: {type}`str | None` name of the dependency group (if any). + dep_template: {type}`str | None` The dep_template to use. + data_exclude: {type}`list[str]` The globs for data attribute exclusion. + enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py files for namespace pkgs. + sdist_filename: {type}`str | None` The filename of the sdist. + namespace_package_files: {type}`list[str]` A list of labels of files whose directories are namespace packages. + filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target names and the glob matches. + copy_files: {type}`dict[str, str]` The mapping between src and dest locations. + copy_executables: {type}`dict[str, str]` The mapping between src and dest locations for executables. + srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion. + data: {type}`list[str]` A list of labels to include as part of the `data` attribute. + visibility: {type}`list[str]` The visibility of the targets. + tags: {type}`list[str]` The tags set on the targets. + **kwargs: Extra args passed to the {obj}`whl_library_targets` and {obj}`whl_library_srcs`. """ + pypi_tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + ] + all_tags = sorted(tags + pypi_tags) + + whl_library_srcs( + name = name, + sdist_filename = sdist_filename, + data_exclude = data_exclude, + srcs_exclude = srcs_exclude, + tags = all_tags, + filegroups = filegroups, + entry_points = entry_points, + visibility = visibility, + data = data, + copy_files = copy_files, + copy_executables = copy_executables, + enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, + namespace_package_files = namespace_package_files, + **kwargs + ) + package_deps = _parse_requires_dist( name = metadata_name, requires_dist = requires_dist, @@ -87,42 +138,22 @@ def whl_library_targets_from_requires( name = name, dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, - entry_points = entry_points, - tags = [ - "pypi_name={}".format(metadata_name), - "pypi_version={}".format(metadata_version), - ], + group_name = group_name, + dep_template = dep_template, + tags = all_tags, **kwargs ) -def _parse_requires_dist( +def whl_library_srcs( *, name, - requires_dist, - excludes, - include, - extras): - return deps( - name = normalize_name(name), - requires_dist = requires_dist, - excludes = excludes, - include = include, - extras = extras, - ) - -def whl_library_targets( - *, - name, - dep_template, sdist_filename = None, data_exclude = [], srcs_exclude = [], tags = [], - dependencies = [], filegroups = None, - dependencies_with_markers = {}, entry_points = {}, - group_name = "", + visibility = ["//visibility:public"], data = [], copy_files = {}, copy_executables = {}, @@ -143,21 +174,13 @@ def whl_library_targets( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. - dep_template: {type}`str` The dep_template to use for dependency - interpolation. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. + visibility: {type}`list[str]` The visibility of the source targets. tags: {type}`list[str]` The tags set on the `py_library`. - dependencies: {type}`list[str]` A list of dependencies. - dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate - in order for the dep to be included. entry_points: {type}`list[dict]` A list of parsed entry point definitions. filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target names and the glob matches. If `None`, defaults will be used. - group_name: {type}`str` name of the dependency group (if any) which - contains this library. If set, this library will behave as a shim - to group implementation rules which will provide simultaneously - installed dependencies which would otherwise form a cycle. copy_executables: {type}`dict[str, str]` The mapping between src and dest locations for the targets. copy_files: {type}`dict[str, str]` The mapping between src and @@ -174,7 +197,6 @@ def whl_library_targets( directories are namespace packages. rules: {type}`struct` A struct with references to rules for creating targets. """ - dependencies = sorted([normalize_name(d) for d in dependencies]) tags = sorted(tags) data = [] + data @@ -233,7 +255,7 @@ def whl_library_targets( native.filegroup( name = filegroup_name, srcs = srcs, - visibility = ["//visibility:public"], + visibility = visibility, ) for src, dest in copy_files.items(): @@ -241,7 +263,7 @@ def whl_library_targets( name = dest + ".copy", src = src, out = dest, - visibility = ["//visibility:public"], + visibility = visibility, ) data.append(dest) for src, dest in copy_executables.items(): @@ -250,10 +272,137 @@ def whl_library_targets( src = src, out = dest, is_executable = True, - visibility = ["//visibility:public"], + visibility = visibility, ) data.append(dest) + if hasattr(native, "filegroup"): + native.filegroup( + name = WHEEL_FILE, + srcs = [name], + visibility = visibility, + ) + + if hasattr(rules, "py_library"): + srcs = native.glob( + ["site-packages/**/*.py"], + exclude = srcs_exclude, + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ) + + # NOTE: pyi files should probably be excluded because they're carried + # by the pyi_srcs attribute. However, historical behavior included + # them in data and some tools currently rely on that. + _data_exclude = [ + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + ] + if sdist_filename: + _data_exclude.append("**/*.dist-info/RECORD") + for item in data_exclude: + if item not in _data_exclude: + _data_exclude.append(item) + + data = data + native.glob( + ["site-packages/**/*"], + exclude = _data_exclude, + allow_empty = True, + ) + + pyi_srcs = native.glob( + ["site-packages/**/*.pyi"], + allow_empty = True, + ) + + if not enable_implicit_namespace_pkgs: + generated_namespace_package_files = select({ + _IS_VENV_SITE_PACKAGES_YES: [], + "//conditions:default": rules.create_inits( + srcs = srcs + data + pyi_srcs, + ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. + root = "site-packages", + ), + }) + namespace_package_files += generated_namespace_package_files + srcs = srcs + generated_namespace_package_files + + # This is done after create_inits() is called so that the data scheme + # files don't have such files created in their directories. + data = data + [DATA_LABEL] + + rules.py_library( + name = PY_SRCS_LABEL, + srcs = srcs, + pyi_srcs = pyi_srcs, + data = data, + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + tags = tags, + visibility = visibility, + experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, + namespace_package_files = namespace_package_files, + ) + +def _parse_requires_dist( + *, + name, + requires_dist, + excludes, + include, + extras): + return deps( + name = normalize_name(name), + requires_dist = requires_dist, + excludes = excludes, + include = include, + extras = extras, + ) + +def whl_library_targets( + *, + name, + dep_template, + tags = [], + dependencies = [], + dependencies_with_markers = {}, + group_name = "", + native = native, + rules = struct( + copy_file = copy_file, + py_binary = py_binary, + py_library = py_library, + venv_entry_point = venv_entry_point, + venv_rewrite_shebang = venv_rewrite_shebang, + env_marker_setting = env_marker_setting, + create_inits = _create_inits, + ), + **_kwargs): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The file to match for including it into the `whl` + filegroup. This may be also parsed to generate extra metadata. + dep_template: {type}`str` The dep_template to use for dependency + interpolation. + tags: {type}`list[str]` The tags set on the `py_library`. + dependencies: {type}`list[str]` A list of dependencies. + dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate + in order for the dep to be included. + group_name: {type}`str` name of the dependency group (if any) which + contains this library. If set, this library will behave as a shim + to group implementation rules which will provide simultaneously + installed dependencies which would otherwise form a cycle. + native: {type}`native` The native struct for overriding in tests. + rules: {type}`struct` A struct with references to rules for creating targets. + **_kwargs: ignored args that are not needed. + """ + dependencies = sorted([normalize_name(d) for d in dependencies]) + tags = sorted(tags) + _config_settings( dependencies_with_markers = dependencies_with_markers, rules = rules, @@ -315,8 +464,9 @@ def whl_library_targets( if hasattr(native, "filegroup"): native.filegroup( name = whl_file_label, - srcs = [name], - data = _deps( + data = [ + WHEEL_FILE, + ] + _deps( deps = dependencies, deps_conditional = deps_conditional, tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), @@ -325,72 +475,23 @@ def whl_library_targets( ) if hasattr(rules, "py_library"): - srcs = native.glob( - ["site-packages/**/*.py"], - exclude = srcs_exclude, - # Empty sources are allowed to support wheels that don't have any - # pure-Python code, e.g. pymssql, which is written in Cython. - allow_empty = True, - ) - - # NOTE: pyi files should probably be excluded because they're carried - # by the pyi_srcs attribute. However, historical behavior included - # them in data and some tools currently rely on that. - _data_exclude = [ - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created - ] - if sdist_filename: - _data_exclude.append("**/*.dist-info/RECORD") - for item in data_exclude: - if item not in _data_exclude: - _data_exclude.append(item) - - data = data + native.glob( - ["site-packages/**/*"], - exclude = _data_exclude, - allow_empty = True, - ) - - pyi_srcs = native.glob( - ["site-packages/**/*.pyi"], - allow_empty = True, - ) - - if not enable_implicit_namespace_pkgs: - generated_namespace_package_files = select({ - _IS_VENV_SITE_PACKAGES_YES: [], - "//conditions:default": rules.create_inits( - srcs = srcs + data + pyi_srcs, - ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. - root = "site-packages", - ), - }) - namespace_package_files += generated_namespace_package_files - srcs = srcs + generated_namespace_package_files - - # This is done after create_inits() is called so that the data scheme - # files don't have such files created in their directories. - data = data + [DATA_LABEL] - rules.py_library( name = py_library_label, - srcs = srcs, - pyi_srcs = pyi_srcs, - data = data, - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["site-packages"], - deps = _deps( + srcs = [ + # We include as srcs to ensure that the (locations :pkg) works as expected. + PY_SRCS_LABEL, + ], + deps = [ + # We include as deps, so that `PyInfo` and friends get propagated as deps. + # not sure if just including it as `srcs` is enough. + PY_SRCS_LABEL, + ] + _deps( deps = dependencies, deps_conditional = deps_conditional, tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), ), tags = tags, visibility = impl_vis, - experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, - namespace_package_files = namespace_package_files, ) def _config_settings(dependencies_with_markers, rules, **kwargs): diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 5bd1d1f549..1f4aac1ba8 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -17,7 +17,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:whl_library_targets.bzl", - "whl_library_targets", + "whl_library_srcs", "whl_library_targets_from_requires", ) # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") @@ -34,9 +34,8 @@ def _test_filegroups(env): return [] return include - whl_library_targets( + whl_library_srcs( name = "", - dep_template = "", native = struct( filegroup = lambda **kwargs: calls.append(kwargs), glob = glob, @@ -63,9 +62,8 @@ def _test_filegroups(env): "visibility": ["//visibility:public"], }, { - "name": "whl", + "name": "whl_file", "srcs": [""], - "data": [], "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -75,9 +73,8 @@ _tests.append(_test_filegroups) def _test_copy(env): calls = [] - whl_library_targets( + whl_library_srcs( name = "", - dep_template = None, filegroups = {}, copy_files = {"file_src": "file_dest"}, copy_executables = {"exec_src": "exec_dest"}, @@ -151,17 +148,23 @@ def _test_whl_and_library_deps_from_requires(env): env.expect.that_collection(filegroup_calls).contains_exactly([ { - "name": "whl", + "name": "whl_file", "srcs": ["foo-0-py3-none-any.whl"], - "data": ["@pypi//bar:whl"] + select({ + "visibility": ["//visibility:public"], + }, + { + "name": "whl", + # NOTE @aignas 2026-07-25: depending on the brackets position one may get different + # results in the expectation. + "data": ["whl_file"] + (["@pypi//bar:whl"] + select({ ":is_include_bar_baz_true": ["@pypi//bar_baz:whl"], "//conditions:default": [], - }), + })), "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items - env.expect.that_collection(py_library_calls).has_size(1) + env.expect.that_collection(py_library_calls).has_size(2) if len(py_library_calls) != 1: return py_library_call = py_library_calls[0] @@ -238,9 +241,8 @@ def _test_sdist_excludes_record(env): m_glob.results.append([]) # data m_glob.results.append([]) # pyi - whl_library_targets( + whl_library_srcs( name = "foo.whl", - dep_template = "@pypi_{name}//:{target}", sdist_filename = "foo.tar.gz", filegroups = {}, native = struct(