From 36e7618af0dda1a3c3436ae9b49c7036a6b1b4f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 16:24:04 +0000 Subject: [PATCH 1/8] Start fix for #505 Assisted-by: Claude Code:claude-opus-5-5 From f0b1977fc005c0a220fda1053be5e9196e372d30 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 16:35:53 +0000 Subject: [PATCH 2/8] Refuse vendoring a Hatch workspace member A vendored scan run from a member of a Hatch 1.16+ workspace wrote {root:uri} references into the member's [project] dependencies and reported success. Hatch's workspace environment reads a member's static dependencies raw, so every sync of the workspace root env then failed with pip's "No connection adapters were found for '{root:uri}/...'" until someone hand-edited the member. Detect a workspace environment declared in an ancestor pyproject.toml or hatch.toml (up to the checkout root) that lists the project, the way Hatch resolves members (globs, path tables, exclude, features and self-referencing extras), and refuse the package with pypi_hatch_unsupported before anything is written, pointing at hosted or agent mode. Dependencies the workspace does not read raw (the member's own env deps, unselected extras, dynamic metadata) are still vendored. Fixes #505 Assisted-by: Claude Code:claude-opus-5-5 --- .../tests/e2e_vex_build/hatch.rs | 85 ++++ .../src/utils/hatch_workspace.rs | 463 ++++++++++++++++++ crates/socket-patch-core/src/utils/mod.rs | 1 + .../src/vendor/pypi_hatch.rs | 94 ++++ docs/testing/hatch.md | 8 +- 5 files changed, 650 insertions(+), 1 deletion(-) create mode 100644 crates/socket-patch-core/src/utils/hatch_workspace.rs diff --git a/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs b/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs index 789e7c522..8ffa40860 100644 --- a/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs +++ b/crates/socket-patch-cli/tests/e2e_vex_build/hatch.rs @@ -740,3 +740,88 @@ fn hatch_existing_env_hosted() { fn hatch_existing_env_vendored() { existing_env_flow(Mode::Vendored); } + +/// #505: a vendored scan run from a member of a Hatch 1.16+ workspace whose +/// root environment installs the member's `[project]` dependencies (read +/// raw, so `{root:uri}` would reach pip unexpanded) is refused with +/// `pypi_hatch_unsupported` and writes nothing; the workspace root's real +/// `hatch env create` still succeeds afterwards. +#[test] +#[ignore = "real Hatch + PyPI; run with --ignored (CI: SOCKET_PATCH_HATCH_E2E_REQUIRED=1)"] +fn hatch_workspace_member_vendored_is_refused() { + let Some(hatch) = hatch() else { return }; + let version = hatch.version.clone(); + if vtuple(&version) < (1, 16) { + eprintln!("SKIP: Hatch {version} has no workspaces"); + return; + } + let what = format!("hatch {version} workspace-member/vendored"); + let tmp = tempfile::tempdir().unwrap(); + let ws = tmp.path().join("ws"); + let member = ws.join("libs/lib1"); + std::fs::create_dir_all(ws.join("src/app")).unwrap(); + std::fs::create_dir_all(member.join("src/lib1")).unwrap(); + std::fs::write(ws.join("src/app/__init__.py"), "").unwrap(); + std::fs::write(member.join("src/lib1/__init__.py"), "").unwrap(); + let root_pyproject = format!( + "{BUILD}[project]\nname = \"app\"\nversion = \"0.1.0\"\ndependencies = []\n\n\ + [tool.hatch.build.targets.wheel]\npackages = [\"src/app\"]\n\n\ + [tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n" + ); + let member_pyproject = format!( + "{BUILD}[project]\nname = \"lib1\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\"]\n\n\ + [tool.hatch.build.targets.wheel]\npackages = [\"src/lib1\"]\n" + ); + std::fs::write(ws.join("pyproject.toml"), &root_pyproject).unwrap(); + std::fs::write(member.join("pyproject.toml"), &member_pyproject).unwrap(); + + let pristine_venv = member.join(".venv"); + if let Err(e) = make_venv(&hatch.uv, &hatch.python(), &pristine_venv, &["six==1.16.0"]) { + skip_or_fail(REQUIRED, &format!("{what}: six==1.16.0 from PyPI: {e}")); + return; + } + let (_, pristine, _) = + six_oracle(&venv_bin(&pristine_venv, "python"), &member).expect("pristine six"); + let patched = [pristine.as_slice(), PATCH_SUFFIX].concat(); + let api = RealApi::start(Mode::Vendored.uuid(), &pristine, &patched); + let scan_envs = vec![("PATH".to_string(), hatch.bin_path())]; + let (code, env, stderr) = + socket_scan(&member, &api, &scan_mode_args(Mode::Vendored), &scan_envs); + assert_ne!(code, Some(0), "{what}: must refuse: {env}\n{stderr}"); + let events: Vec<&serde_json::Value> = env["events"].as_array().into_iter().flatten().collect(); + assert!( + events + .iter() + .any(|e| e["errorCode"] == "pypi_hatch_unsupported" + && e.to_string().contains("Hatch workspace")), + "{what}: {env}\n{stderr}" + ); + assert_eq!( + std::fs::read_to_string(member.join("pyproject.toml")).unwrap(), + member_pyproject, + "{what}: member pyproject must be untouched" + ); + assert_eq!( + std::fs::read_to_string(ws.join("pyproject.toml")).unwrap(), + root_pyproject, + "{what}: root pyproject must be untouched" + ); + let _ = std::fs::remove_dir_all(&pristine_venv); + + // The workspace root env still syncs from the untouched member. + let case = tmp.path().join("case"); + let envs = hatch.case_env(&case); + let out = hatch.run(&ws, &envs, &["env", "create"]); + assert!( + out.status.success(), + "{what}: root `hatch env create`: {}", + String::from_utf8_lossy(&out.stderr) + ); + record( + "hatch", + &version, + "workspace-member/vendored", + "refused", + "pass", + ); +} diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs new file mode 100644 index 000000000..d59e8748c --- /dev/null +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -0,0 +1,463 @@ +//! Hatch workspaces (Hatch 1.16+, #505): an environment's +//! `workspace.members` installs each member editable together with the +//! member's `[project]` dependencies and the optional dependencies of the +//! member's `features`. For a member whose dependencies are static, Hatch +//! reads those strings from the member's `pyproject.toml` raw, without +//! context formatting, so a `{root:uri}` reference there reaches pip +//! unexpanded and every sync of the workspace environment fails. This +//! module finds the workspace environments that consume a project that +//! way, read the way Hatch 1.18's `Workspace.members` resolves them. + +use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::path::{Component, Path, PathBuf}; + +use toml_edit::{DocumentMut, Item, Value}; + +use crate::crawlers::python_crawler::canonicalize_pypi_name; +use crate::utils::hatch::HATCH_FILES; +use crate::vendor::common::pep508_name; + +/// A workspace environment that installs a member's dependencies raw. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RawConsumer { + /// The directory of the project declaring the workspace. + pub root: PathBuf, + /// The environment whose `workspace.members` lists the member. + pub env: String, +} + +/// The first workspace environment, declared in an ancestor of `member` +/// (up to the nearest directory holding `.git`), that installs `name` from +/// the member's static `[project]` dependencies or the optional +/// dependencies of the features it selects. `member_files` are the +/// member's [`HATCH_FILES`]. `None` when no workspace reads `name` raw, +/// including when the member's dependencies are dynamic: Hatch then builds +/// the metadata through the backend, which expands `{root:uri}`. +pub fn raw_consumer( + member: &Path, + member_files: &BTreeMap, + name: &str, +) -> Option { + let document: DocumentMut = member_files.get(HATCH_FILES[0])?.parse().ok()?; + let project = document.get("project")?; + let dynamic = strings(project.get("dynamic")); + if dynamic + .iter() + .any(|field| matches!(*field, "dependencies" | "optional-dependencies")) + { + return None; + } + let member = lexical_absolute(member)?; + for ancestor in member.ancestors().skip(1) { + for (env, features) in workspace_features(ancestor, &member) { + if consumed(project, &features).contains(&canonicalize_pypi_name(name)) { + return Some(RawConsumer { + root: ancestor.to_path_buf(), + env, + }); + } + } + if ancestor.join(".git").exists() { + break; + } + } + None +} + +/// `path` made absolute and normalized lexically (`.` dropped, `..` popped), +/// the way Python's `os.path.abspath` / `normpath` read it: Hatch compares +/// member paths without resolving symlinks. +fn lexical_absolute(path: &Path) -> Option { + let absolute = std::path::absolute(path).ok()?; + let mut out = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + other => out.push(other), + } + } + Some(out) +} + +fn strings(item: Option<&Item>) -> Vec<&str> { + item.and_then(Item::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect() +} + +fn parse(dir: &Path, file: &str) -> Option { + std::fs::read_to_string(dir.join(file)).ok()?.parse().ok() +} + +/// For each environment of the project at `root` whose workspace lists +/// `member`: the environment name and the member's selected features +/// (normalized). The environments are hatch.toml's `envs` when it has that +/// key, else pyproject's `[tool.hatch.envs]` (Hatch merges an external +/// config by top-level key). +fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> { + let external = parse(root, HATCH_FILES[1]).filter(|doc| doc.contains_key("envs")); + let project = match external { + Some(_) => None, + None => parse(root, HATCH_FILES[0]), + }; + let envs = match (&external, &project) { + (Some(doc), _) => doc.get("envs"), + (None, Some(doc)) => doc + .get("tool") + .and_then(|tool| tool.get("hatch")) + .and_then(|hatch| hatch.get("envs")), + (None, None) => None, + }; + let Some(envs) = envs.and_then(Item::as_table_like) else { + return Vec::new(); + }; + let mut found = Vec::new(); + for (env, config) in envs.iter() { + let Some(workspace) = config.get("workspace") else { + continue; + }; + let exclude = strings(workspace.get("exclude")); + let Some(members) = workspace.get("members").and_then(Item::as_array) else { + continue; + }; + for entry in members.iter() { + let (path, features) = match entry { + Value::String(path) => (path.value().as_str(), Vec::new()), + Value::InlineTable(table) => { + let Some(path) = table.get("path").and_then(Value::as_str) else { + continue; + }; + let features = table + .get("features") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(canonicalize_pypi_name) + .collect(); + (path, features) + } + _ => continue, + }; + if lists(root, path, &exclude, member) { + found.push((env.to_string(), features)); + } + } + } + found +} + +/// Whether the member spec `spec` of the workspace at `root` lists +/// `member`, as Hatch's `find_members` resolves it: the spec is joined to +/// the root and normalized, the part below the root and the spec's shared +/// prefix is matched one component at a time (a component with `*`, `?` or +/// `[` as an fnmatch pattern Python searches for, so it may match a suffix +/// of the directory name), and a member any `exclude` pattern matches, by +/// its path relative to that prefix or its absolute path, is dropped. A +/// spec using context formatting (`{root}`, `{env:..}`) is not resolved. +fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { + if spec.is_empty() || spec.contains('{') { + return false; + } + let Some(target) = lexical_absolute(&root.join(spec)) else { + return false; + }; + let prefix: PathBuf = root + .components() + .zip(target.components()) + .take_while(|(a, b)| a == b) + .map(|(a, _)| a) + .collect(); + let (Ok(pattern), Ok(relative)) = (target.strip_prefix(&prefix), member.strip_prefix(&prefix)) + else { + return false; + }; + let pattern: Vec = pattern.iter().map(|c| c.to_string_lossy().into()).collect(); + let relative: Vec = relative + .iter() + .map(|c| c.to_string_lossy().into()) + .collect(); + if pattern.len() != relative.len() + || !pattern.iter().zip(&relative).all(|(pattern, name)| { + if pattern.contains(['*', '?', '[']) { + let name: Vec = name.chars().collect(); + let pattern: Vec = pattern.chars().collect(); + (0..=name.len()).any(|start| fnmatch(&pattern, &name[start..])) + } else { + pattern == name + } + }) + { + return false; + } + let relative = relative.join(std::path::MAIN_SEPARATOR_STR); + let absolute = member.to_string_lossy(); + !exclude.iter().any(|pattern| { + let pattern: Vec = pattern.chars().collect(); + [relative.as_str(), &absolute] + .iter() + .any(|text| fnmatch(&pattern, &text.chars().collect::>())) + }) +} + +/// Python's `fnmatch.fnmatchcase` over the whole of `text`: `*` matches +/// any run (separators included), `?` one character, `[..]` a class +/// (`[!..]` negated, `a-z` ranges); an unclosed `[` is literal. +fn fnmatch(pattern: &[char], text: &[char]) -> bool { + match pattern.first() { + None => text.is_empty(), + Some('*') => (0..=text.len()).any(|skip| fnmatch(&pattern[1..], &text[skip..])), + Some('?') => !text.is_empty() && fnmatch(&pattern[1..], &text[1..]), + Some('[') => match class(&pattern[1..]) { + Some((set, rest)) => { + text.first().is_some_and(|&c| set(c)) && fnmatch(&pattern[rest + 1..], &text[1..]) + } + None => text.first() == Some(&'[') && fnmatch(&pattern[1..], &text[1..]), + }, + Some(&c) => text.first() == Some(&c) && fnmatch(&pattern[1..], &text[1..]), + } +} + +/// The class starting after a `[`: its predicate and the length consumed +/// up to and including the closing `]`, or `None` when it never closes. +#[allow(clippy::type_complexity)] +fn class(body: &[char]) -> Option<(Box bool>, usize)> { + let negated = body.first() == Some(&'!'); + let start = usize::from(negated); + // A `]` right after the opening (or `[!`) is a member, not the close. + let close = body + .iter() + .enumerate() + .skip(start + 1) + .find(|(_, c)| **c == ']') + .map(|(i, _)| i)?; + let items = body[start..close].to_vec(); + let mut ranges = Vec::new(); + let mut i = 0; + while i < items.len() { + if i + 2 < items.len() && items[i + 1] == '-' { + ranges.push((items[i], items[i + 2])); + i += 3; + } else { + ranges.push((items[i], items[i])); + i += 1; + } + } + let set = move |c: char| ranges.iter().any(|(lo, hi)| *lo <= c && c <= *hi) != negated; + Some((Box::new(set), close + 1)) +} + +/// The canonical names of the dependencies a workspace installs from a +/// member's static `[project]` table: its `dependencies`, plus the optional +/// dependencies of `features`, following the member's references to its +/// own extras (`lib[other]`) the way Hatch's `resolve_extras` does. +fn consumed(project: &Item, features: &[String]) -> BTreeSet { + let own = project + .get("name") + .and_then(Item::as_str) + .map(canonicalize_pypi_name); + let groups: BTreeMap> = project + .get("optional-dependencies") + .and_then(Item::as_table_like) + .into_iter() + .flat_map(|table| table.iter()) + .map(|(key, group)| (canonicalize_pypi_name(key), strings(Some(group)))) + .collect(); + let mut names = BTreeSet::new(); + let mut seen = BTreeSet::new(); + let mut queue: VecDeque<&str> = strings(project.get("dependencies")).into(); + for feature in features { + if seen.insert(feature.clone()) { + queue.extend(groups.get(feature).into_iter().flatten()); + } + } + while let Some(spec) = queue.pop_front() { + let name = canonicalize_pypi_name(pep508_name(spec)); + if own.as_deref() != Some(name.as_str()) { + names.insert(name); + continue; + } + for extra in extras(spec) { + if seen.insert(extra.clone()) { + queue.extend(groups.get(&extra).into_iter().flatten()); + } + } + } + names +} + +/// The normalized extras of a PEP 508 spec (`name[a, b]`). +fn extras(spec: &str) -> Vec { + let rest = spec.trim_start()[pep508_name(spec).len()..].trim_start(); + rest.strip_prefix('[') + .and_then(|rest| rest.split_once(']')) + .map(|(list, _)| { + list.split(',') + .map(str::trim) + .filter(|extra| !extra.is_empty()) + .map(canonicalize_pypi_name) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + const ROOT: &str = "[project]\nname = \"app\"\nversion = \"0.1.0\"\n"; + const MEMBER: &str = + "[project]\nname = \"lib1\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\"]\n\ + [project.optional-dependencies]\nsec = [\"urllib3==1.26.18\"]\n\ + all = [\"lib1[sec]\", \"idna==3.4\"]\n"; + + /// A workspace at `/ws` (a git checkout) with `root` as its + /// pyproject and `hatch` as its hatch.toml, and the member `member`. + fn layout(root: &str, hatch: Option<&str>, member: &str) -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let ws = temp.path().join("ws"); + std::fs::create_dir_all(ws.join(".git")).unwrap(); + std::fs::write(ws.join("pyproject.toml"), root).unwrap(); + if let Some(hatch) = hatch { + std::fs::write(ws.join("hatch.toml"), hatch).unwrap(); + } + let dir = ws.join(member); + std::fs::create_dir_all(&dir).unwrap(); + (temp, dir) + } + + fn consumer(dir: &Path, member: &str, name: &str) -> Option { + let files = BTreeMap::from([("pyproject.toml".to_string(), member.to_string())]); + raw_consumer(dir, &files, name).map(|found| found.env) + } + + /// #505: a member listed by a glob, a plain path or a `{path = ..}` + /// table is consumed for its `[project]` dependencies; a project the + /// workspace does not list, or none at all, is not. + #[test] + fn members_listed_by_glob_path_or_table() { + for members in [ + "[\"libs/*\"]", + "[\"libs/lib1\"]", + "[\"./libs/../libs/lib1/\"]", + "[{ path = \"libs/lib1\" }]", + "[\"libs/l?b[0-9]\"]", + ] { + let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"); + let (_temp, dir) = layout(&root, None, "libs/lib1"); + assert_eq!( + consumer(&dir, MEMBER, "six").as_deref(), + Some("default"), + "{members}" + ); + assert_eq!(consumer(&dir, MEMBER, "Six").as_deref(), Some("default")); + } + for members in ["[\"other/*\"]", "[\"libs\"]", "[\"libs/*/x\"]", "[]"] { + let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"); + let (_temp, dir) = layout(&root, None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six"), None, "{members}"); + } + let (_temp, dir) = layout(ROOT, None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six"), None); + } + + /// #505: optional dependencies are consumed only through the member's + /// `features`, including extras the feature reaches through the + /// member's references to itself. + #[test] + fn optional_dependencies_follow_member_features() { + let with = |features: &str| { + format!( + "{ROOT}[tool.hatch.envs.default.workspace]\n\ + members = [{{ path = \"libs/lib1\", features = {features} }}]\n" + ) + }; + let (_temp, dir) = layout(&with("[\"sec\"]"), None, "libs/lib1"); + assert_eq!( + consumer(&dir, MEMBER, "urllib3").as_deref(), + Some("default") + ); + assert_eq!(consumer(&dir, MEMBER, "idna"), None); + let (_temp, dir) = layout(&with("[\"ALL\"]"), None, "libs/lib1"); + assert_eq!( + consumer(&dir, MEMBER, "urllib3").as_deref(), + Some("default") + ); + assert_eq!(consumer(&dir, MEMBER, "idna").as_deref(), Some("default")); + let (_temp, dir) = layout(&with("[]"), None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + } + + /// #505: a workspace declared in hatch.toml counts, and a hatch.toml + /// `envs` table hides pyproject's environments. + #[test] + fn hatch_toml_workspaces_replace_pyproject_envs() { + let hatch = "[envs.test.workspace]\nmembers = [\"libs/*\"]\n"; + let (_temp, dir) = layout(ROOT, Some(hatch), "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("test")); + let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"); + let (_temp, dir) = layout(&root, Some("[envs.lint]\n"), "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six"), None); + let (_temp, dir) = layout(&root, Some("[build]\n"), "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + } + + /// `exclude` drops a listed member; dynamic member dependencies are + /// built by the backend, which expands `{root:uri}`. + #[test] + fn excluded_and_dynamic_members_are_not_raw() { + let root = format!( + "{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\nexclude = [\"libs/lib1\"]\n" + ); + let (_temp, dir) = layout(&root, None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six"), None); + let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"); + let (_temp, dir) = layout(&root, None, "libs/lib1"); + let dynamic = MEMBER.replace( + "version = ", + "dynamic = [\"optional-dependencies\"]\nversion = ", + ); + assert_eq!(consumer(&dir, &dynamic, "six"), None); + } + + /// The walk stops at the checkout root: a workspace above `.git` is + /// not read. + #[test] + fn ancestors_stop_at_the_checkout_root() { + let temp = tempfile::tempdir().unwrap(); + std::fs::write( + temp.path().join("pyproject.toml"), + format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"repo/lib1\"]\n"), + ) + .unwrap(); + let dir = temp.path().join("repo/lib1"); + std::fs::create_dir_all(&dir).unwrap(); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + std::fs::create_dir_all(temp.path().join("repo/.git")).unwrap(); + assert_eq!(consumer(&dir, MEMBER, "six"), None); + } + + #[test] + fn fnmatch_reads_python_patterns() { + let m = |p: &str, t: &str| { + fnmatch( + &p.chars().collect::>(), + &t.chars().collect::>(), + ) + }; + assert!(m("pkg-*", "pkg-a")); + assert!(m("*", "a/b")); + assert!(m("[!x]y", "ay")); + assert!(!m("[!x]y", "xy")); + assert!(m("[a-c]", "b")); + assert!(m("[]]", "]")); + assert!(m("a[", "a[")); + assert!(!m("a?", "a")); + } +} diff --git a/crates/socket-patch-core/src/utils/mod.rs b/crates/socket-patch-core/src/utils/mod.rs index 127f2c121..06ea44531 100644 --- a/crates/socket-patch-core/src/utils/mod.rs +++ b/crates/socket-patch-core/src/utils/mod.rs @@ -40,3 +40,4 @@ pub(crate) mod uv_workspace; pub(crate) mod workspace_globs; pub mod hatch; +pub mod hatch_workspace; diff --git a/crates/socket-patch-core/src/vendor/pypi_hatch.rs b/crates/socket-patch-core/src/vendor/pypi_hatch.rs index ec7f35b47..45777b83b 100644 --- a/crates/socket-patch-core/src/vendor/pypi_hatch.rs +++ b/crates/socket-patch-core/src/vendor/pypi_hatch.rs @@ -42,6 +42,7 @@ pub(super) async fn load( let files = read_files(root).await?; require_pip_installer()?; require_expandable_root(root)?; + require_no_raw_workspace(root, &files, name).await?; let prefix = format!("{{root:uri}}/.socket/vendor/pypi/{uuid}/"); let state = super::state::load_state_shared(root) .await @@ -117,6 +118,7 @@ pub(super) async fn preflight(root: &Path, name: &str) -> Result<(), Failure> { let files = read_files(root).await?; require_pip_installer()?; require_expandable_root(root)?; + require_no_raw_workspace(root, &files, name).await?; hatch::require_pip_installer(&files).map_err(|error| ("pypi_hatch_unsupported", error))?; if hatch::has_environment_dependency(&files, name) { require_environment_context_support(root).await?; @@ -138,6 +140,33 @@ fn require_expandable_root(root: &Path) -> Result<(), Failure> { } } +/// Refuse a member of a Hatch workspace whose environment installs `name` +/// from this project's `[project]` table (#505): Hatch reads a member's +/// static dependencies raw, so the workspace root's environments would get +/// an unexpanded `{root:uri}` and fail every sync. +async fn require_no_raw_workspace( + root: &Path, + files: &BTreeMap, + name: &str, +) -> Result<(), Failure> { + let (member, files, owned) = (root.to_path_buf(), files.clone(), name.to_owned()); + let consumer = crate::utils::fs::run_blocking(move || { + crate::utils::hatch_workspace::raw_consumer(&member, &files, &owned) + }) + .await; + match consumer { + Some(consumer) => Err(( + "pypi_hatch_unsupported", + format!( + "this project is a member of the Hatch workspace at {} (environment {:?}), which installs {name} from this project's dependencies without expanding {{root:uri}}, so a vendored reference would break that workspace's environments; use hosted or agent mode", + consumer.root.display(), + consumer.env, + ), + )), + None => Ok(()), + } +} + fn require_pip_installer() -> Result<(), Failure> { if std::env::var("HATCH_ENV_TYPE_VIRTUAL_UV_PATH").is_ok_and(|path| !path.is_empty()) { return Err(("pypi_hatch_unsupported", "vendored Hatch wheels require the pip installer: uv does not enforce local wheel fragment hashes".into())); @@ -516,6 +545,71 @@ mod tests { } } + /// #505: `load` and `preflight` refuse a member whose `[project]` + /// dependency (or a selected feature's) a Hatch workspace in an + /// ancestor pyproject.toml or hatch.toml installs raw, before writing; + /// the member's own environment dependencies stay vendorable. + #[tokio::test] + async fn member_consumed_by_a_hatch_workspace_is_refused() { + let member = + "[project]\nname = \"lib1\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\"]\n\ + [project.optional-dependencies]\nsec = [\"one==1\"]\n\ + [tool.hatch.envs.default]\ndependencies = [\"two==2\"]\n"; + let root = "[project]\nname = \"app\"\nversion = \"0.1.0\"\n"; + for (pyproject, hatch, refused) in [ + ( + format!("{root}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"), + None, + &["six"][..], + ), + ( + format!( + "{root}[tool.hatch.envs.default.workspace]\n\ + members = [{{ path = \"libs/lib1\", features = [\"sec\"] }}]\n" + ), + None, + &["six", "one"][..], + ), + ( + root.to_string(), + Some("[envs.default.workspace]\nmembers = [\"libs/lib1\"]\n"), + &["six"][..], + ), + ] { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir(temp.path().join(".git")).unwrap(); + std::fs::write(temp.path().join("pyproject.toml"), &pyproject).unwrap(); + if let Some(hatch) = hatch { + std::fs::write(temp.path().join("hatch.toml"), hatch).unwrap(); + } + let dir = temp.path().join("libs/lib1"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("pyproject.toml"), member).unwrap(); + for name in ["six", "one"] { + let load = load(&dir, name, "1", UUID).await.map(|_| ()); + let preflight = preflight(&dir, name).await; + if refused.contains(&name) { + let Err((code, detail)) = load else { + panic!("{pyproject}: {name} must be refused"); + }; + assert_eq!(code, "pypi_hatch_unsupported"); + assert!( + detail.contains("Hatch workspace") && detail.contains("{root:uri}"), + "{detail}" + ); + assert_eq!(preflight.map_err(|(code, _)| code), Err(code)); + } else { + assert!(load.is_ok(), "{pyproject}: {name}: {load:?}"); + assert!(preflight.is_ok(), "{pyproject}: {name}"); + } + } + assert_eq!( + std::fs::read_to_string(dir.join("pyproject.toml")).unwrap(), + member + ); + } + } + #[tokio::test] async fn recorded_pin_drift_missing_artifact_and_ledgerless_source() { let temp = tempfile::tempdir().unwrap(); diff --git a/docs/testing/hatch.md b/docs/testing/hatch.md index 248089223..ad756637d 100644 --- a/docs/testing/hatch.md +++ b/docs/testing/hatch.md @@ -37,7 +37,13 @@ A range, transitive-only declaration, dynamic dependency metadata, custom environment plugin, source table or conditional override is refused before writing. Use agent mode (`scan --mode agent` + `socket-patch apply` in CI) for these shapes. Hosted PEP 735 groups are supported; vendored groups are refused because Hatch does not expand -`{root:uri}` within dependency groups. Unknown direct sources require an +`{root:uri}` within dependency groups. Vendoring a member of a Hatch 1.16+ +workspace is refused the same way when a workspace environment declared in +an ancestor `pyproject.toml` or `hatch.toml` (up to the checkout root) +installs the package from the member's static `[project]` dependencies or +the optional dependencies of the member's `features`: Hatch reads those +strings raw, so the workspace environments would get an unexpanded +`{root:uri}` (#505). Unknown direct sources require an explicit revert before patching. A hosted scan re-pins socket-patch's own earlier hosted reference (same package and version on the patch server) when the grant rotates or a newer patch supersedes it. A vendored scan From ce0ef5902b119ca0c5df694f7250263eabfe3b8f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 17:22:14 +0000 Subject: [PATCH 3/8] Find sibling Hatch workspace roots and more specs Review follow-ups for the workspace-member refusal: - look for a workspace root anywhere in the checkout, since Hatch lets a root list a member beside it through a ../ spec - read members written as an array of tables and specs using the {root} and {home} context fields - stop at the checkout root even when the member is that root - match fnmatch patterns in linear time, and fold case and separators on Windows the way Python's fnmatch does - look a selected feature up by its exact key, as Hatch does - tell a user whose member was vendored by an earlier run how to undo it - share the hatch.toml/pyproject env-table rule with the planner Assisted-by: Claude Code:claude-opus-5-5 --- crates/socket-patch-core/src/utils/hatch.rs | 34 +- .../src/utils/hatch_workspace.rs | 492 +++++++++++------- .../src/vendor/pypi_hatch.rs | 15 +- 3 files changed, 345 insertions(+), 196 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch.rs b/crates/socket-patch-core/src/utils/hatch.rs index 914e7c535..46108180b 100644 --- a/crates/socket-patch-core/src/utils/hatch.rs +++ b/crates/socket-patch-core/src/utils/hatch.rs @@ -138,6 +138,26 @@ pub(crate) fn dependency_specs<'d>( specs } +/// The `envs` item Hatch reads and the [`HATCH_FILES`] entry holding it: +/// hatch.toml's when that document has an `envs` key (Hatch merges an +/// external config by TOP-LEVEL key, so it replaces pyproject's whole +/// table), else pyproject's `[tool.hatch.envs]`. +pub(crate) fn environments<'d>( + pyproject: Option<&'d DocumentMut>, + hatch_toml: Option<&'d DocumentMut>, +) -> (&'static str, Option<&'d Item>) { + match hatch_toml.filter(|d| d.contains_key("envs")) { + Some(doc) => (HATCH_FILES[1], doc.get("envs")), + None => ( + HATCH_FILES[0], + pyproject + .and_then(|d| d.get("tool")) + .and_then(|t| t.get("hatch")) + .and_then(|h| h.get("envs")), + ), + } +} + /// The Hatch environment tables: `envs` of the hatch.toml document, else of /// pyproject's `[tool.hatch]` (Hatch merges an external config by TOP-LEVEL /// key, so a hatch.toml `envs` key — whatever its value — replaces @@ -146,18 +166,8 @@ fn environment_specs<'d>( pyproject: Option<&'d DocumentMut>, hatch_toml: Option<&'d DocumentMut>, ) -> Vec> { - let (file, hatch) = match hatch_toml.filter(|d| d.contains_key("envs")) { - Some(doc) => (HATCH_FILES[1], Some(doc.as_item())), - None => ( - HATCH_FILES[0], - pyproject - .and_then(|d| d.get("tool")) - .and_then(|t| t.get("hatch")), - ), - }; - hatch - .and_then(|h| h.get("envs")) - .and_then(Item::as_table_like) + let (file, envs) = environments(pyproject, hatch_toml); + envs.and_then(Item::as_table_like) .into_iter() .flat_map(|envs| envs.iter()) .flat_map(|(_, env)| { diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index d59e8748c..3c15970f2 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -9,14 +9,17 @@ //! way, read the way Hatch 1.18's `Workspace.members` resolves them. use std::collections::{BTreeMap, BTreeSet, VecDeque}; -use std::path::{Component, Path, PathBuf}; +use std::path::{Path, PathBuf}; -use toml_edit::{DocumentMut, Item, Value}; +use toml_edit::{DocumentMut, Item, TableLike, Value}; use crate::crawlers::python_crawler::canonicalize_pypi_name; -use crate::utils::hatch::HATCH_FILES; +use crate::utils::hatch::{environments, HATCH_FILES}; use crate::vendor::common::pep508_name; +/// How deep below the checkout root a workspace root is looked for. +const MAX_DEPTH: usize = 8; + /// A workspace environment that installs a member's dependencies raw. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RawConsumer { @@ -26,60 +29,84 @@ pub struct RawConsumer { pub env: String, } -/// The first workspace environment, declared in an ancestor of `member` -/// (up to the nearest directory holding `.git`), that installs `name` from -/// the member's static `[project]` dependencies or the optional -/// dependencies of the features it selects. `member_files` are the -/// member's [`HATCH_FILES`]. `None` when no workspace reads `name` raw, -/// including when the member's dependencies are dynamic: Hatch then builds -/// the metadata through the backend, which expands `{root:uri}`. -pub fn raw_consumer( - member: &Path, - member_files: &BTreeMap, - name: &str, -) -> Option { - let document: DocumentMut = member_files.get(HATCH_FILES[0])?.parse().ok()?; +/// The first workspace environment that installs `name` from the member's +/// static `[project]` dependencies, or from the optional dependencies of +/// the features it selects, read from `member_pyproject`. Workspace roots +/// are looked for in the member's ancestors and, when the member sits in a +/// git checkout, in every project of that checkout (a root can list a +/// sibling through a `../` spec); the walk never leaves the checkout. +/// `None` when no workspace reads `name` raw, including when the member's +/// dependencies are dynamic: Hatch then builds the metadata through the +/// backend, which expands `{root:uri}`. +pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option { + let document: DocumentMut = member_pyproject.parse().ok()?; let project = document.get("project")?; - let dynamic = strings(project.get("dynamic")); - if dynamic + if strings(project.get("dynamic")) .iter() .any(|field| matches!(*field, "dependencies" | "optional-dependencies")) { return None; } - let member = lexical_absolute(member)?; - for ancestor in member.ancestors().skip(1) { - for (env, features) in workspace_features(ancestor, &member) { - if consumed(project, &features).contains(&canonicalize_pypi_name(name)) { - return Some(RawConsumer { - root: ancestor.to_path_buf(), - env, - }); - } - } - if ancestor.join(".git").exists() { - break; - } - } - None + let member = absolute(member)?; + let name = canonicalize_pypi_name(name); + roots(&member).into_iter().find_map(|root| { + workspace_features(&root, &member) + .into_iter() + .find(|(_, features)| consumed(project, features).contains(&name)) + .map(|(env, _)| RawConsumer { root, env }) + }) } -/// `path` made absolute and normalized lexically (`.` dropped, `..` popped), -/// the way Python's `os.path.abspath` / `normpath` read it: Hatch compares -/// member paths without resolving symlinks. -fn lexical_absolute(path: &Path) -> Option { - let absolute = std::path::absolute(path).ok()?; - let mut out = PathBuf::new(); - for component in absolute.components() { - match component { - Component::CurDir => {} - Component::ParentDir => { - out.pop(); - } - other => out.push(other), +/// `path` made absolute and normalized lexically, the way Python's +/// `os.path.abspath` reads it: Hatch compares member paths without +/// resolving symlinks. +fn absolute(path: &Path) -> Option { + crate::utils::relpath::normalize_lexically(&std::path::absolute(path).ok()?) +} + +/// The directories that may declare a workspace listing `member`: its +/// ancestors up to the checkout root (every ancestor outside a checkout), +/// then the other projects of the checkout. +fn roots(member: &Path) -> Vec { + let checkout = member + .ancestors() + .find(|dir| dir.join(".git").exists()) + .map(Path::to_path_buf); + let mut roots: Vec = member + .ancestors() + .skip(1) + .take_while(|dir| checkout.as_deref().is_none_or(|top| dir.starts_with(top))) + .map(Path::to_path_buf) + .collect(); + let Some(checkout) = checkout else { + return roots; + }; + let walk = ignore::WalkBuilder::new(&checkout) + .max_depth(Some(MAX_DEPTH)) + .require_git(false) + .filter_entry(|entry| { + !matches!( + entry.file_name().to_str(), + Some("node_modules" | "__pypackages__" | "site-packages") + ) + }) + .build(); + for entry in walk.flatten() { + let path = entry.path(); + let Some(dir) = path.parent() else { continue }; + if !HATCH_FILES + .iter() + .any(|file| path.file_name() == Some(file.as_ref())) + || dir == member + || roots.iter().any(|root| root == dir) + { + continue; + } + if std::fs::read_to_string(path).is_ok_and(|text| text.contains("workspace")) { + roots.push(dir.to_path_buf()); } } - Some(out) + roots } fn strings(item: Option<&Item>) -> Vec<&str> { @@ -94,25 +121,24 @@ fn parse(dir: &Path, file: &str) -> Option { std::fs::read_to_string(dir.join(file)).ok()?.parse().ok() } +/// One `workspace.members` entry: its path spec and normalized features. +fn member_entry(table: &dyn TableLike) -> Option<(String, Vec)> { + let path = table.get("path")?.as_str()?.to_owned(); + let features = strings(table.get("features")) + .into_iter() + .map(canonicalize_pypi_name) + .collect(); + Some((path, features)) +} + /// For each environment of the project at `root` whose workspace lists /// `member`: the environment name and the member's selected features -/// (normalized). The environments are hatch.toml's `envs` when it has that -/// key, else pyproject's `[tool.hatch.envs]` (Hatch merges an external -/// config by top-level key). +/// (normalized). Members are a string array, an array of inline tables or +/// an array of tables (`[[...workspace.members]]`). fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> { - let external = parse(root, HATCH_FILES[1]).filter(|doc| doc.contains_key("envs")); - let project = match external { - Some(_) => None, - None => parse(root, HATCH_FILES[0]), - }; - let envs = match (&external, &project) { - (Some(doc), _) => doc.get("envs"), - (None, Some(doc)) => doc - .get("tool") - .and_then(|tool| tool.get("hatch")) - .and_then(|hatch| hatch.get("envs")), - (None, None) => None, - }; + let external = parse(root, HATCH_FILES[1]); + let project = parse(root, HATCH_FILES[0]); + let (_, envs) = environments(project.as_ref(), external.as_ref()); let Some(envs) = envs.and_then(Item::as_table_like) else { return Vec::new(); }; @@ -122,29 +148,23 @@ fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> continue; }; let exclude = strings(workspace.get("exclude")); - let Some(members) = workspace.get("members").and_then(Item::as_array) else { - continue; + let entries: Vec<(String, Vec)> = match workspace.get("members") { + Some(Item::Value(Value::Array(members))) => members + .iter() + .filter_map(|entry| match entry { + Value::String(path) => Some((path.value().clone(), Vec::new())), + Value::InlineTable(table) => member_entry(table), + _ => None, + }) + .collect(), + Some(Item::ArrayOfTables(members)) => members + .iter() + .filter_map(|table| member_entry(table)) + .collect(), + _ => continue, }; - for entry in members.iter() { - let (path, features) = match entry { - Value::String(path) => (path.value().as_str(), Vec::new()), - Value::InlineTable(table) => { - let Some(path) = table.get("path").and_then(Value::as_str) else { - continue; - }; - let features = table - .get("features") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .map(canonicalize_pypi_name) - .collect(); - (path, features) - } - _ => continue, - }; - if lists(root, path, &exclude, member) { + for (path, features) in entries { + if lists(root, &path, &exclude, member) { found.push((env.to_string(), features)); } } @@ -152,19 +172,29 @@ fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> found } +/// A member spec with the context fields Hatch formats in it (`{root}`, +/// `{home}`) filled in; `None` when another field remains. +fn expand(root: &Path, spec: &str) -> Option { + let mut spec = spec.replace("{root}", &root.to_string_lossy()); + if spec.contains("{home}") { + let home = crate::utils::fs::home_dir()?; + spec = spec.replace("{home}", &home.to_string_lossy()); + } + (!spec.contains('{')).then_some(spec) +} + /// Whether the member spec `spec` of the workspace at `root` lists /// `member`, as Hatch's `find_members` resolves it: the spec is joined to /// the root and normalized, the part below the root and the spec's shared /// prefix is matched one component at a time (a component with `*`, `?` or /// `[` as an fnmatch pattern Python searches for, so it may match a suffix /// of the directory name), and a member any `exclude` pattern matches, by -/// its path relative to that prefix or its absolute path, is dropped. A -/// spec using context formatting (`{root}`, `{env:..}`) is not resolved. +/// its path relative to that prefix or its absolute path, is dropped. fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { - if spec.is_empty() || spec.contains('{') { + let Some(spec) = expand(root, spec).filter(|spec| !spec.is_empty()) else { return false; - } - let Some(target) = lexical_absolute(&root.join(spec)) else { + }; + let Some(target) = absolute(&root.join(spec)) else { return false; }; let prefix: PathBuf = root @@ -185,9 +215,7 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { if pattern.len() != relative.len() || !pattern.iter().zip(&relative).all(|(pattern, name)| { if pattern.contains(['*', '?', '[']) { - let name: Vec = name.chars().collect(); - let pattern: Vec = pattern.chars().collect(); - (0..=name.len()).any(|start| fnmatch(&pattern, &name[start..])) + fnmatch(&format!("*{pattern}"), name) } else { pattern == name } @@ -197,83 +225,137 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { } let relative = relative.join(std::path::MAIN_SEPARATOR_STR); let absolute = member.to_string_lossy(); + // `fnmatch.fnmatch` applies `os.path.normcase` to both sides. + let normcase = |text: &str| { + if cfg!(windows) { + text.to_lowercase().replace('/', "\\") + } else { + text.to_owned() + } + }; !exclude.iter().any(|pattern| { - let pattern: Vec = pattern.chars().collect(); + let pattern = normcase(pattern); [relative.as_str(), &absolute] .iter() - .any(|text| fnmatch(&pattern, &text.chars().collect::>())) + .any(|text| fnmatch(&pattern, &normcase(text))) }) } +enum Token { + Star, + Any, + Class(Vec<(char, char)>, bool), + Literal(char), +} + /// Python's `fnmatch.fnmatchcase` over the whole of `text`: `*` matches /// any run (separators included), `?` one character, `[..]` a class -/// (`[!..]` negated, `a-z` ranges); an unclosed `[` is literal. -fn fnmatch(pattern: &[char], text: &[char]) -> bool { - match pattern.first() { - None => text.is_empty(), - Some('*') => (0..=text.len()).any(|skip| fnmatch(&pattern[1..], &text[skip..])), - Some('?') => !text.is_empty() && fnmatch(&pattern[1..], &text[1..]), - Some('[') => match class(&pattern[1..]) { - Some((set, rest)) => { - text.first().is_some_and(|&c| set(c)) && fnmatch(&pattern[rest + 1..], &text[1..]) +/// (`[!..]` negated, `a-z` ranges); an unclosed `[` is literal. Matched +/// greedily, backtracking only to the last `*`, so time stays linear in +/// the number of stars. +fn fnmatch(pattern: &str, text: &str) -> bool { + let tokens = tokenize(&pattern.chars().collect::>()); + let text: Vec = text.chars().collect(); + let (mut p, mut t) = (0, 0); + let mut resume: Option<(usize, usize)> = None; + while t < text.len() { + let step = match tokens.get(p) { + Some(Token::Star) => { + resume = Some((p, t)); + p += 1; + continue; } - None => text.first() == Some(&'[') && fnmatch(&pattern[1..], &text[1..]), - }, - Some(&c) => text.first() == Some(&c) && fnmatch(&pattern[1..], &text[1..]), + Some(Token::Any) => true, + Some(Token::Class(ranges, negated)) => { + ranges.iter().any(|(lo, hi)| (*lo..=*hi).contains(&text[t])) != *negated + } + Some(Token::Literal(c)) => *c == text[t], + None => false, + }; + if step { + p += 1; + t += 1; + } else if let Some((star, from)) = resume { + p = star + 1; + t = from + 1; + resume = Some((star, from + 1)); + } else { + return false; + } } + tokens[p..].iter().all(|token| matches!(token, Token::Star)) } -/// The class starting after a `[`: its predicate and the length consumed -/// up to and including the closing `]`, or `None` when it never closes. -#[allow(clippy::type_complexity)] -fn class(body: &[char]) -> Option<(Box bool>, usize)> { - let negated = body.first() == Some(&'!'); - let start = usize::from(negated); - // A `]` right after the opening (or `[!`) is a member, not the close. - let close = body - .iter() - .enumerate() - .skip(start + 1) - .find(|(_, c)| **c == ']') - .map(|(i, _)| i)?; - let items = body[start..close].to_vec(); - let mut ranges = Vec::new(); +fn tokenize(pattern: &[char]) -> Vec { + let mut tokens = Vec::new(); let mut i = 0; - while i < items.len() { - if i + 2 < items.len() && items[i + 1] == '-' { - ranges.push((items[i], items[i + 2])); - i += 3; - } else { - ranges.push((items[i], items[i])); - i += 1; + while i < pattern.len() { + match pattern[i] { + '*' => tokens.push(Token::Star), + '?' => tokens.push(Token::Any), + '[' => { + let negated = pattern.get(i + 1) == Some(&'!'); + let start = i + 1 + usize::from(negated); + // A `]` right after the opening (or `[!`) is a member. + match (start + 1..pattern.len()).find(|&j| pattern[j] == ']') { + Some(close) => { + let items = &pattern[start..close]; + let mut ranges = Vec::new(); + let mut k = 0; + while k < items.len() { + if k + 2 < items.len() && items[k + 1] == '-' { + ranges.push((items[k], items[k + 2])); + k += 3; + } else { + ranges.push((items[k], items[k])); + k += 1; + } + } + tokens.push(Token::Class(ranges, negated)); + i = close; + } + None => tokens.push(Token::Literal('[')), + } + } + c => tokens.push(Token::Literal(c)), } + i += 1; } - let set = move |c: char| ranges.iter().any(|(lo, hi)| *lo <= c && c <= *hi) != negated; - Some((Box::new(set), close + 1)) + tokens } /// The canonical names of the dependencies a workspace installs from a /// member's static `[project]` table: its `dependencies`, plus the optional -/// dependencies of `features`, following the member's references to its -/// own extras (`lib[other]`) the way Hatch's `resolve_extras` does. +/// dependencies of `features`. As in Hatch, a selected feature names its +/// table key exactly, and the member's references to its own extras +/// (`lib[other]`) are followed through `resolve_extras`, which also +/// accepts a normalized key. fn consumed(project: &Item, features: &[String]) -> BTreeSet { let own = project .get("name") .and_then(Item::as_str) .map(canonicalize_pypi_name); - let groups: BTreeMap> = project + let groups: BTreeMap<&str, Vec<&str>> = project .get("optional-dependencies") .and_then(Item::as_table_like) .into_iter() .flat_map(|table| table.iter()) - .map(|(key, group)| (canonicalize_pypi_name(key), strings(Some(group)))) + .map(|(key, group)| (key, strings(Some(group)))) .collect(); + let normalized = |extra: &str| { + groups.get(extra).or_else(|| { + groups + .iter() + .find(|(key, _)| canonicalize_pypi_name(key) == extra) + .map(|(_, group)| group) + }) + }; let mut names = BTreeSet::new(); let mut seen = BTreeSet::new(); let mut queue: VecDeque<&str> = strings(project.get("dependencies")).into(); for feature in features { if seen.insert(feature.clone()) { - queue.extend(groups.get(feature).into_iter().flatten()); + queue.extend(groups.get(feature.as_str()).into_iter().flatten()); } } while let Some(spec) = queue.pop_front() { @@ -284,7 +366,7 @@ fn consumed(project: &Item, features: &[String]) -> BTreeSet { } for extra in extras(spec) { if seen.insert(extra.clone()) { - queue.extend(groups.get(&extra).into_iter().flatten()); + queue.extend(normalized(&extra).into_iter().flatten()); } } } @@ -314,31 +396,47 @@ mod tests { const MEMBER: &str = "[project]\nname = \"lib1\"\nversion = \"0.1.0\"\ndependencies = [\"six==1.16.0\"]\n\ [project.optional-dependencies]\nsec = [\"urllib3==1.26.18\"]\n\ - all = [\"lib1[sec]\", \"idna==3.4\"]\n"; + all = [\"lib1[sec]\", \"idna==3.4\"]\nSpaced_Out = [\"certifi==1\"]\n"; - /// A workspace at `/ws` (a git checkout) with `root` as its - /// pyproject and `hatch` as its hatch.toml, and the member `member`. - fn layout(root: &str, hatch: Option<&str>, member: &str) -> (tempfile::TempDir, PathBuf) { + /// A git checkout at `/ws` with `root` as the pyproject and + /// `hatch` as the hatch.toml of the workspace at `/ws/`, and + /// the member directory `member`. + fn layout_at( + at: &str, + root: &str, + hatch: Option<&str>, + member: &str, + ) -> (tempfile::TempDir, PathBuf) { let temp = tempfile::tempdir().unwrap(); let ws = temp.path().join("ws"); std::fs::create_dir_all(ws.join(".git")).unwrap(); - std::fs::write(ws.join("pyproject.toml"), root).unwrap(); + let declaring = ws.join(at); + std::fs::create_dir_all(&declaring).unwrap(); + std::fs::write(declaring.join("pyproject.toml"), root).unwrap(); if let Some(hatch) = hatch { - std::fs::write(ws.join("hatch.toml"), hatch).unwrap(); + std::fs::write(declaring.join("hatch.toml"), hatch).unwrap(); } let dir = ws.join(member); std::fs::create_dir_all(&dir).unwrap(); (temp, dir) } + fn layout(root: &str, hatch: Option<&str>, member: &str) -> (tempfile::TempDir, PathBuf) { + layout_at("", root, hatch, member) + } + fn consumer(dir: &Path, member: &str, name: &str) -> Option { - let files = BTreeMap::from([("pyproject.toml".to_string(), member.to_string())]); - raw_consumer(dir, &files, name).map(|found| found.env) + raw_consumer(dir, member, name).map(|found| found.env) } - /// #505: a member listed by a glob, a plain path or a `{path = ..}` - /// table is consumed for its `[project]` dependencies; a project the - /// workspace does not list, or none at all, is not. + fn workspace(members: &str) -> String { + format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = {members}\n") + } + + /// #505: a member listed by a glob, a plain path, a `{path = ..}` + /// table, an array of tables or a `{root}` spec is consumed for its + /// `[project]` dependencies; a project the workspace does not list, or + /// none at all, is not. #[test] fn members_listed_by_glob_path_or_table() { for members in [ @@ -347,9 +445,12 @@ mod tests { "[\"./libs/../libs/lib1/\"]", "[{ path = \"libs/lib1\" }]", "[\"libs/l?b[0-9]\"]", + // Python searches a component pattern, so it may match a + // suffix of the directory name. + "[\"libs/b[1]\"]", + "[\"{root}/libs/lib1\"]", ] { - let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"); - let (_temp, dir) = layout(&root, None, "libs/lib1"); + let (_temp, dir) = layout(&workspace(&members), None, "libs/lib1"); assert_eq!( consumer(&dir, MEMBER, "six").as_deref(), Some("default"), @@ -357,25 +458,49 @@ mod tests { ); assert_eq!(consumer(&dir, MEMBER, "Six").as_deref(), Some("default")); } - for members in ["[\"other/*\"]", "[\"libs\"]", "[\"libs/*/x\"]", "[]"] { - let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"); - let (_temp, dir) = layout(&root, None, "libs/lib1"); + let tables = + format!("{ROOT}[[tool.hatch.envs.default.workspace.members]]\npath = \"libs/lib1\"\n"); + let (_temp, dir) = layout(&tables, None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + for members in [ + "[\"other/*\"]", + "[\"libs\"]", + "[\"libs/*/x\"]", + "[]", + "[\"{env:X}/libs/lib1\"]", + ] { + let (_temp, dir) = layout(&workspace(members), None, "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "six"), None, "{members}"); } let (_temp, dir) = layout(ROOT, None, "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "six"), None); } + /// #505 (review): a workspace root beside the member, listing it + /// through a `../` spec, is found anywhere in the checkout. + #[test] + fn sibling_roots_listing_the_member_through_parent_specs() { + let (_temp, dir) = layout_at( + "apps/web", + &workspace("[\"../../libs/*\"]"), + None, + "libs/lib1", + ); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + let (_temp, dir) = layout_at("apps/web", &workspace("[\"../x/*\"]"), None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "six"), None); + } + /// #505: optional dependencies are consumed only through the member's /// `features`, including extras the feature reaches through the - /// member's references to itself. + /// member's references to itself; a selected feature names its key + /// exactly. #[test] fn optional_dependencies_follow_member_features() { let with = |features: &str| { - format!( - "{ROOT}[tool.hatch.envs.default.workspace]\n\ - members = [{{ path = \"libs/lib1\", features = {features} }}]\n" - ) + workspace(&format!( + "[{{ path = \"libs/lib1\", features = {features} }}]" + )) }; let (_temp, dir) = layout(&with("[\"sec\"]"), None, "libs/lib1"); assert_eq!( @@ -392,6 +517,9 @@ mod tests { let (_temp, dir) = layout(&with("[]"), None, "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + // Hatch looks the normalized `spaced-out` up as a raw key. + let (_temp, dir) = layout(&with("[\"Spaced_Out\"]"), None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "certifi"), None); } /// #505: a workspace declared in hatch.toml counts, and a hatch.toml @@ -401,7 +529,7 @@ mod tests { let hatch = "[envs.test.workspace]\nmembers = [\"libs/*\"]\n"; let (_temp, dir) = layout(ROOT, Some(hatch), "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("test")); - let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"); + let root = workspace("[\"libs/*\"]"); let (_temp, dir) = layout(&root, Some("[envs.lint]\n"), "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "six"), None); let (_temp, dir) = layout(&root, Some("[build]\n"), "libs/lib1"); @@ -412,13 +540,10 @@ mod tests { /// built by the backend, which expands `{root:uri}`. #[test] fn excluded_and_dynamic_members_are_not_raw() { - let root = format!( - "{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\nexclude = [\"libs/lib1\"]\n" - ); + let root = format!("{}exclude = [\"libs/lib1\"]\n", workspace("[\"libs/*\"]")); let (_temp, dir) = layout(&root, None, "libs/lib1"); assert_eq!(consumer(&dir, MEMBER, "six"), None); - let root = format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"); - let (_temp, dir) = layout(&root, None, "libs/lib1"); + let (_temp, dir) = layout(&workspace("[\"libs/*\"]"), None, "libs/lib1"); let dynamic = MEMBER.replace( "version = ", "dynamic = [\"optional-dependencies\"]\nversion = ", @@ -426,14 +551,14 @@ mod tests { assert_eq!(consumer(&dir, &dynamic, "six"), None); } - /// The walk stops at the checkout root: a workspace above `.git` is - /// not read. + /// The walk stops at the checkout root, including when the member is + /// that root: a workspace above `.git` is not read. #[test] fn ancestors_stop_at_the_checkout_root() { let temp = tempfile::tempdir().unwrap(); std::fs::write( temp.path().join("pyproject.toml"), - format!("{ROOT}[tool.hatch.envs.default.workspace]\nmembers = [\"repo/lib1\"]\n"), + workspace("[\"repo/lib1\", \"top\"]"), ) .unwrap(); let dir = temp.path().join("repo/lib1"); @@ -441,23 +566,28 @@ mod tests { assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); std::fs::create_dir_all(temp.path().join("repo/.git")).unwrap(); assert_eq!(consumer(&dir, MEMBER, "six"), None); + let top = temp.path().join("top"); + std::fs::create_dir_all(&top).unwrap(); + assert_eq!(consumer(&top, MEMBER, "six").as_deref(), Some("default")); + std::fs::create_dir_all(top.join(".git")).unwrap(); + assert_eq!(consumer(&top, MEMBER, "six"), None); } #[test] fn fnmatch_reads_python_patterns() { - let m = |p: &str, t: &str| { - fnmatch( - &p.chars().collect::>(), - &t.chars().collect::>(), - ) - }; - assert!(m("pkg-*", "pkg-a")); - assert!(m("*", "a/b")); - assert!(m("[!x]y", "ay")); - assert!(!m("[!x]y", "xy")); - assert!(m("[a-c]", "b")); - assert!(m("[]]", "]")); - assert!(m("a[", "a[")); - assert!(!m("a?", "a")); + assert!(fnmatch("pkg-*", "pkg-a")); + assert!(fnmatch("*", "a/b")); + assert!(fnmatch("*a*b", "xxaxxb")); + assert!(!fnmatch("*a*b", "xxaxxbc")); + assert!(fnmatch("[!x]y", "ay")); + assert!(!fnmatch("[!x]y", "xy")); + assert!(fnmatch("[a-c]", "b")); + assert!(fnmatch("[]]", "]")); + assert!(fnmatch("a[", "a[")); + assert!(!fnmatch("a?", "a")); + assert!(fnmatch("a**", "a")); + // Many stars against a long miss stays fast. + let text = "x".repeat(200); + assert!(!fnmatch("*/*/*/*/*/*/*/*/legacy", &text)); } } diff --git a/crates/socket-patch-core/src/vendor/pypi_hatch.rs b/crates/socket-patch-core/src/vendor/pypi_hatch.rs index 45777b83b..2b852be88 100644 --- a/crates/socket-patch-core/src/vendor/pypi_hatch.rs +++ b/crates/socket-patch-core/src/vendor/pypi_hatch.rs @@ -149,18 +149,27 @@ async fn require_no_raw_workspace( files: &BTreeMap, name: &str, ) -> Result<(), Failure> { - let (member, files, owned) = (root.to_path_buf(), files.clone(), name.to_owned()); + let Some(pyproject) = files.get(hatch::HATCH_FILES[0]).cloned() else { + return Ok(()); + }; + let vendored = pyproject.contains("{root:uri}/.socket/vendor/"); + let (member, owned) = (root.to_path_buf(), name.to_owned()); let consumer = crate::utils::fs::run_blocking(move || { - crate::utils::hatch_workspace::raw_consumer(&member, &files, &owned) + crate::utils::hatch_workspace::raw_consumer(&member, &pyproject, &owned) }) .await; match consumer { Some(consumer) => Err(( "pypi_hatch_unsupported", format!( - "this project is a member of the Hatch workspace at {} (environment {:?}), which installs {name} from this project's dependencies without expanding {{root:uri}}, so a vendored reference would break that workspace's environments; use hosted or agent mode", + "this project is a member of the Hatch workspace at {} (environment {:?}), which installs {name} from this project's dependencies without expanding {{root:uri}}, so a vendored reference would break that workspace's environments; use hosted or agent mode{}", consumer.root.display(), consumer.env, + if vendored { + " (undo references an earlier run vendored here with `socket-patch vendor --revert`)" + } else { + "" + }, ), )), None => Ok(()), From 40c302461b74933b701ba904f13667bd842b32c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 17:40:03 +0000 Subject: [PATCH 4/8] Bound Hatch workspace reads like repo lookups Read workspace candidates through the non-blocking regular-file reader, so a FIFO at a pyproject.toml or hatch.toml path can't hang a vendored scan, and walk ancestors with the shared repository lookup, which stops at the checkout root, never enters the home directory and honours GIT_CEILING_DIRECTORIES. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/utils/hatch_workspace.rs | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index 3c15970f2..8fdfed360 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -14,7 +14,9 @@ use std::path::{Path, PathBuf}; use toml_edit::{DocumentMut, Item, TableLike, Value}; use crate::crawlers::python_crawler::canonicalize_pypi_name; +use crate::utils::fs::read_regular_to_string_sync; use crate::utils::hatch::{environments, HATCH_FILES}; +use crate::utils::repo_root; use crate::vendor::common::pep508_name; /// How deep below the checkout root a workspace root is looked for. @@ -47,7 +49,10 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option { return None; } - let member = absolute(member)?; + // The same spelling as the canonical walk in [`roots`]. + let member = std::fs::canonicalize(member) + .ok() + .or_else(|| absolute(member))?; let name = canonicalize_pypi_name(name); roots(&member).into_iter().find_map(|root| { workspace_features(&root, &member) @@ -58,27 +63,19 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option } /// `path` made absolute and normalized lexically, the way Python's -/// `os.path.abspath` reads it: Hatch compares member paths without -/// resolving symlinks. +/// `os.path.abspath` reads it. fn absolute(path: &Path) -> Option { crate::utils::relpath::normalize_lexically(&std::path::absolute(path).ok()?) } -/// The directories that may declare a workspace listing `member`: its -/// ancestors up to the checkout root (every ancestor outside a checkout), -/// then the other projects of the checkout. +/// The directories that may declare a workspace listing `member` (a +/// canonical path): its ancestors as far as a repository lookup goes +/// ([`repo_root::search_dirs`]: through the checkout root, never into the +/// home directory or past a `GIT_CEILING_DIRECTORIES` entry), then the +/// other projects of that checkout. fn roots(member: &Path) -> Vec { - let checkout = member - .ancestors() - .find(|dir| dir.join(".git").exists()) - .map(Path::to_path_buf); - let mut roots: Vec = member - .ancestors() - .skip(1) - .take_while(|dir| checkout.as_deref().is_none_or(|top| dir.starts_with(top))) - .map(Path::to_path_buf) - .collect(); - let Some(checkout) = checkout else { + let mut roots: Vec = repo_root::search_dirs(member).into_iter().skip(1).collect(); + let Some(checkout) = repo_root::find_git_repo(member).map(|repo| repo.root) else { return roots; }; let walk = ignore::WalkBuilder::new(&checkout) @@ -102,7 +99,7 @@ fn roots(member: &Path) -> Vec { { continue; } - if std::fs::read_to_string(path).is_ok_and(|text| text.contains("workspace")) { + if read_regular_to_string_sync(path).is_ok_and(|text| text.contains("workspace")) { roots.push(dir.to_path_buf()); } } @@ -118,7 +115,10 @@ fn strings(item: Option<&Item>) -> Vec<&str> { } fn parse(dir: &Path, file: &str) -> Option { - std::fs::read_to_string(dir.join(file)).ok()?.parse().ok() + read_regular_to_string_sync(&dir.join(file)) + .ok()? + .parse() + .ok() } /// One `workspace.members` entry: its path spec and normalized features. @@ -573,6 +573,29 @@ mod tests { assert_eq!(consumer(&top, MEMBER, "six"), None); } + /// A FIFO squatting an ancestor's or a checkout project's + /// pyproject.toml / hatch.toml is skipped, never opened blocking. + #[cfg(unix)] + #[test] + fn fifos_in_place_of_workspace_files_never_block() { + let (temp, dir) = layout(ROOT, None, "libs/lib1"); + let ws = temp.path().join("ws"); + std::fs::remove_file(ws.join("pyproject.toml")).unwrap(); + std::fs::create_dir_all(ws.join("apps/web")).unwrap(); + for fifo in [ + ws.join("pyproject.toml"), + ws.join("hatch.toml"), + ws.join("apps/web/pyproject.toml"), + ] { + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap(); + assert!(status.success()); + } + assert_eq!(consumer(&dir, MEMBER, "six"), None); + } + #[test] fn fnmatch_reads_python_patterns() { assert!(fnmatch("pkg-*", "pkg-a")); From c37fc342ba91b15b83f066e7b18c88ceb61627e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 25 Oct 2026 17:56:04 +0000 Subject: [PATCH 5/8] Match Hatch members reached through symlinks Resolve each workspace member spec by listing directories the way Hatch's find_members does, following symlinked directories, and compare what it finds with the scanned project by real path. Roots are looked for above both the spelled and the real path, so a member Hatch reaches through a symlink is refused whichever path the scan was given. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/utils/hatch_workspace.rs | 135 ++++++++++++------ 1 file changed, 92 insertions(+), 43 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index 8fdfed360..427604934 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -49,12 +49,13 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option { return None; } - // The same spelling as the canonical walk in [`roots`]. - let member = std::fs::canonicalize(member) - .ok() - .or_else(|| absolute(member))?; + // Hatch reaches a member through symlinks without resolving them, so + // roots are looked for above both spellings and found members are + // compared by their real path. + let lexical = absolute(member)?; + let member = std::fs::canonicalize(&lexical).unwrap_or_else(|_| lexical.clone()); let name = canonicalize_pypi_name(name); - roots(&member).into_iter().find_map(|root| { + roots(&member, &lexical).into_iter().find_map(|root| { workspace_features(&root, &member) .into_iter() .find(|(_, features)| consumed(project, features).contains(&name)) @@ -68,14 +69,28 @@ fn absolute(path: &Path) -> Option { crate::utils::relpath::normalize_lexically(&std::path::absolute(path).ok()?) } -/// The directories that may declare a workspace listing `member` (a -/// canonical path): its ancestors as far as a repository lookup goes -/// ([`repo_root::search_dirs`]: through the checkout root, never into the -/// home directory or past a `GIT_CEILING_DIRECTORIES` entry), then the -/// other projects of that checkout. -fn roots(member: &Path) -> Vec { - let mut roots: Vec = repo_root::search_dirs(member).into_iter().skip(1).collect(); - let Some(checkout) = repo_root::find_git_repo(member).map(|repo| repo.root) else { +/// The directories that may declare a workspace listing the member at +/// `real` (its canonical path) or `lexical` (as spelled): their ancestors +/// as far as a repository lookup goes ([`repo_root::search_dirs`]: through +/// the checkout root, never into the home directory or past a +/// `GIT_CEILING_DIRECTORIES` entry), then the other projects of the +/// checkout. +fn roots(real: &Path, lexical: &Path) -> Vec { + let home = repo_root::home(); + let mut roots: Vec = repo_root::search_dirs(real).into_iter().skip(1).collect(); + let at_checkout = lexical.join(".git").exists(); + for dir in lexical.ancestors().skip(1).take_while(|_| !at_checkout) { + if home.as_deref() == Some(dir) { + break; + } + if !roots.iter().any(|root| root == dir) { + roots.push(dir.to_path_buf()); + } + if dir.join(".git").exists() { + break; + } + } + let Some(checkout) = repo_root::find_git_repo(real).map(|repo| repo.root) else { return roots; }; let walk = ignore::WalkBuilder::new(&checkout) @@ -94,7 +109,7 @@ fn roots(member: &Path) -> Vec { if !HATCH_FILES .iter() .any(|file| path.file_name() == Some(file.as_ref())) - || dir == member + || dir == real || roots.iter().any(|root| root == dir) { continue; @@ -184,12 +199,14 @@ fn expand(root: &Path, spec: &str) -> Option { } /// Whether the member spec `spec` of the workspace at `root` lists -/// `member`, as Hatch's `find_members` resolves it: the spec is joined to -/// the root and normalized, the part below the root and the spec's shared -/// prefix is matched one component at a time (a component with `*`, `?` or -/// `[` as an fnmatch pattern Python searches for, so it may match a suffix -/// of the directory name), and a member any `exclude` pattern matches, by -/// its path relative to that prefix or its absolute path, is dropped. +/// `member` (a canonical path), as Hatch's `find_members` resolves it: the +/// spec is joined to the root and normalized, and the part below the root +/// and the spec's shared prefix is matched one directory level at a time +/// (a component with `*`, `?` or `[` as an fnmatch pattern Python searches +/// for, so it may match a suffix of the directory name; symlinked +/// directories are followed). A found directory whose real path is +/// `member` lists it unless an `exclude` pattern matches its path relative +/// to that prefix or its absolute path. fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { let Some(spec) = expand(root, spec).filter(|spec| !spec.is_empty()) else { return false; @@ -203,28 +220,32 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { .take_while(|(a, b)| a == b) .map(|(a, _)| a) .collect(); - let (Ok(pattern), Ok(relative)) = (target.strip_prefix(&prefix), member.strip_prefix(&prefix)) - else { + let Ok(pattern) = target.strip_prefix(&prefix) else { return false; }; - let pattern: Vec = pattern.iter().map(|c| c.to_string_lossy().into()).collect(); - let relative: Vec = relative - .iter() - .map(|c| c.to_string_lossy().into()) - .collect(); - if pattern.len() != relative.len() - || !pattern.iter().zip(&relative).all(|(pattern, name)| { - if pattern.contains(['*', '?', '[']) { - fnmatch(&format!("*{pattern}"), name) - } else { - pattern == name + let mut found = vec![prefix.clone()]; + for component in pattern.iter().map(|c| c.to_string_lossy().into_owned()) { + let globbed = component.contains(['*', '?', '[']); + let search = format!("*{component}"); + let mut next = Vec::new(); + for dir in &found { + let Ok(entries) = std::fs::read_dir(dir) else { + continue; + }; + for entry in entries.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + let matched = if globbed { + fnmatch(&search, &name) + } else { + name == component + }; + if matched && entry.path().is_dir() { + next.push(entry.path()); + } } - }) - { - return false; + } + found = next; } - let relative = relative.join(std::path::MAIN_SEPARATOR_STR); - let absolute = member.to_string_lossy(); // `fnmatch.fnmatch` applies `os.path.normcase` to both sides. let normcase = |text: &str| { if cfg!(windows) { @@ -233,11 +254,21 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { text.to_owned() } }; - !exclude.iter().any(|pattern| { - let pattern = normcase(pattern); - [relative.as_str(), &absolute] - .iter() - .any(|text| fnmatch(&pattern, &normcase(text))) + found.iter().any(|path| { + if std::fs::canonicalize(path).ok().as_deref() != Some(member) { + return false; + } + let relative = path + .strip_prefix(&prefix) + .map(|rel| rel.to_string_lossy().into_owned()) + .unwrap_or_default(); + let absolute = path.to_string_lossy(); + !exclude.iter().any(|pattern| { + let pattern = normcase(pattern); + [relative.as_str(), &absolute] + .iter() + .any(|text| fnmatch(&pattern, &normcase(text))) + }) }) } @@ -573,6 +604,24 @@ mod tests { assert_eq!(consumer(&top, MEMBER, "six"), None); } + /// Bugbot: a member Hatch reaches through a symlinked directory is + /// still listed, whichever spelling the scan was given. + #[cfg(unix)] + #[test] + fn members_reached_through_symlinks_are_listed() { + let (temp, real) = layout(&workspace("[\"libs/*\"]"), None, "vendor/lib1"); + let ws = temp.path().join("ws"); + std::fs::create_dir_all(ws.join("libs")).unwrap(); + std::os::unix::fs::symlink(&real, ws.join("libs/lib1")).unwrap(); + assert_eq!(consumer(&real, MEMBER, "six").as_deref(), Some("default")); + assert_eq!( + consumer(&ws.join("libs/lib1"), MEMBER, "six").as_deref(), + Some("default") + ); + let (_temp, real) = layout(&workspace("[\"other/*\"]"), None, "vendor/lib1"); + assert_eq!(consumer(&real, MEMBER, "six"), None); + } + /// A FIFO squatting an ancestor's or a checkout project's /// pyproject.toml / hatch.toml is skipped, never opened blocking. #[cfg(unix)] From 5650d4631a2f13c3476c0fa3f1bc3ae7e482b2c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:23:38 +0000 Subject: [PATCH 6/8] Count member extras named across a Hatch workspace Hatch's all_dependencies_complex expands a workspace member's raw optional dependencies whenever any of the environment's requirement strings names the member with an extra: the root's own dependencies, its environments', or another member's (lib[s3]). Those groups reach pip raw, so a vendored {root:uri} there breaks the workspace just like a selected feature does. Count those extras as consumed. Also say in the Hatch notes that a sibling root listing the member through ../ is found as well. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/utils/hatch_workspace.rs | 147 +++++++++++++++--- docs/testing/hatch.md | 10 +- 2 files changed, 133 insertions(+), 24 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index 427604934..8c743c59c 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -55,8 +55,13 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option let lexical = absolute(member)?; let member = std::fs::canonicalize(&lexical).unwrap_or_else(|_| lexical.clone()); let name = canonicalize_pypi_name(name); + let own = project + .get("name") + .and_then(Item::as_str) + .map(canonicalize_pypi_name) + .unwrap_or_default(); roots(&member, &lexical).into_iter().find_map(|root| { - workspace_features(&root, &member) + workspace_features(&root, &member, &own) .into_iter() .find(|(_, features)| consumed(project, features).contains(&name)) .map(|(env, _)| RawConsumer { root, env }) @@ -148,9 +153,11 @@ fn member_entry(table: &dyn TableLike) -> Option<(String, Vec)> { /// For each environment of the project at `root` whose workspace lists /// `member`: the environment name and the member's selected features -/// (normalized). Members are a string array, an array of inline tables or +/// (normalized): those the entry selects, plus every extra of `own` (the +/// member's canonical name) the workspace's requirement strings name. +/// Members are a string array, an array of inline tables or /// an array of tables (`[[...workspace.members]]`). -fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> { +fn workspace_features(root: &Path, member: &Path, own: &str) -> Vec<(String, Vec)> { let external = parse(root, HATCH_FILES[1]); let project = parse(root, HATCH_FILES[0]); let (_, envs) = environments(project.as_ref(), external.as_ref()); @@ -178,15 +185,76 @@ fn workspace_features(root: &Path, member: &Path) -> Vec<(String, Vec)> .collect(), _ => continue, }; - for (path, features) in entries { - if lists(root, &path, &exclude, member) { - found.push((env.to_string(), features)); + let listed: Vec<(Vec, Vec)> = entries + .into_iter() + .map(|(path, features)| (members_of(root, &path, &exclude), features)) + .collect(); + let real = |path: &PathBuf| std::fs::canonicalize(path).ok(); + for (members, features) in &listed { + if !members + .iter() + .any(|path| real(path).as_deref() == Some(member)) + { + continue; } + // `all_dependencies_complex` expands every member's raw extras + // that any of the environment's requirement strings name as + // `[extra]`: the root's own and its environments', + // and the other members' (#505 review). + let mut features = features.clone(); + let mut texts: Vec = project.iter().chain(&external).cloned().collect(); + texts.extend( + listed + .iter() + .flat_map(|(paths, _)| paths) + .filter(|path| real(path).as_deref() != Some(member)) + .filter_map(|path| parse(path, HATCH_FILES[0])), + ); + for document in &texts { + features.extend(referenced_extras(document.as_item(), own)); + } + found.push((env.to_string(), features)); } } found } +/// The normalized extras of `own` that any requirement string anywhere in +/// `item` names as `own[extra]`. +fn referenced_extras(item: &Item, own: &str) -> Vec { + fn walk(value: &Value, own: &str, out: &mut Vec) { + match value { + Value::String(text) => { + let spec = text.value(); + if canonicalize_pypi_name(pep508_name(spec)) == own { + out.extend(extras(spec)); + } + } + Value::Array(array) => array.iter().for_each(|value| walk(value, own, out)), + Value::InlineTable(table) => table.iter().for_each(|(_, value)| walk(value, own, out)), + _ => {} + } + } + let mut out = Vec::new(); + match item { + Item::Value(value) => walk(value, own, &mut out), + Item::Table(table) => { + for (_, item) in table.iter() { + out.extend(referenced_extras(item, own)); + } + } + Item::ArrayOfTables(tables) => { + for table in tables.iter() { + for (_, item) in table.iter() { + out.extend(referenced_extras(item, own)); + } + } + } + Item::None => {} + } + out +} + /// A member spec with the context fields Hatch formats in it (`{root}`, /// `{home}`) filled in; `None` when another field remains. fn expand(root: &Path, spec: &str) -> Option { @@ -198,21 +266,20 @@ fn expand(root: &Path, spec: &str) -> Option { (!spec.contains('{')).then_some(spec) } -/// Whether the member spec `spec` of the workspace at `root` lists -/// `member` (a canonical path), as Hatch's `find_members` resolves it: the +/// The members (directories, as reached) the member spec `spec` of the +/// workspace at `root` lists, as Hatch's `find_members` resolves it: the /// spec is joined to the root and normalized, and the part below the root /// and the spec's shared prefix is matched one directory level at a time /// (a component with `*`, `?` or `[` as an fnmatch pattern Python searches /// for, so it may match a suffix of the directory name; symlinked -/// directories are followed). A found directory whose real path is -/// `member` lists it unless an `exclude` pattern matches its path relative -/// to that prefix or its absolute path. -fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { +/// directories are followed), less any an `exclude` pattern matches by +/// its path relative to that prefix or its absolute path. +fn members_of(root: &Path, spec: &str, exclude: &[&str]) -> Vec { let Some(spec) = expand(root, spec).filter(|spec| !spec.is_empty()) else { - return false; + return Vec::new(); }; let Some(target) = absolute(&root.join(spec)) else { - return false; + return Vec::new(); }; let prefix: PathBuf = root .components() @@ -221,7 +288,7 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { .map(|(a, _)| a) .collect(); let Ok(pattern) = target.strip_prefix(&prefix) else { - return false; + return Vec::new(); }; let mut found = vec![prefix.clone()]; for component in pattern.iter().map(|c| c.to_string_lossy().into_owned()) { @@ -254,10 +321,7 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { text.to_owned() } }; - found.iter().any(|path| { - if std::fs::canonicalize(path).ok().as_deref() != Some(member) { - return false; - } + found.retain(|path| { let relative = path .strip_prefix(&prefix) .map(|rel| rel.to_string_lossy().into_owned()) @@ -269,7 +333,8 @@ fn lists(root: &Path, spec: &str, exclude: &[&str], member: &Path) -> bool { .iter() .any(|text| fnmatch(&pattern, &normcase(text))) }) - }) + }); + found } enum Token { @@ -553,6 +618,48 @@ mod tests { assert_eq!(consumer(&dir, MEMBER, "certifi"), None); } + /// #505 review: an extra of the member that the workspace's own + /// requirement strings name (`lib1[sec]` in an environment, the root + /// project or another member) is installed raw too, transitively. + #[test] + fn extras_named_across_the_workspace_are_consumed() { + let members = "[\"libs/*\"]"; + let env = format!( + "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[sec]\"]\n\ + [tool.hatch.envs.default.workspace]\nmembers = {members}\n" + ); + let (_temp, dir) = layout(&env, None, "libs/lib1"); + assert_eq!( + consumer(&dir, MEMBER, "urllib3").as_deref(), + Some("default") + ); + assert_eq!(consumer(&dir, MEMBER, "idna"), None); + let root = ROOT.replace("version", "dependencies = [\"lib1[all]\"]\nversion"); + let (_temp, dir) = layout( + &format!("{root}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"), + None, + "libs/lib1", + ); + assert_eq!( + consumer(&dir, MEMBER, "urllib3").as_deref(), + Some("default") + ); + assert_eq!(consumer(&dir, MEMBER, "idna").as_deref(), Some("default")); + let (temp, dir) = layout(&workspace(members), None, "libs/lib1"); + let sibling = temp.path().join("ws/libs/lib2"); + std::fs::create_dir_all(&sibling).unwrap(); + std::fs::write( + sibling.join("pyproject.toml"), + "[project]\nname = \"lib2\"\nversion = \"1\"\ndependencies = [\"LIB1[Sec]\"]\n", + ) + .unwrap(); + assert_eq!( + consumer(&dir, MEMBER, "urllib3").as_deref(), + Some("default") + ); + assert_eq!(consumer(&dir, MEMBER, "certifi"), None); + } + /// #505: a workspace declared in hatch.toml counts, and a hatch.toml /// `envs` table hides pyproject's environments. #[test] diff --git a/docs/testing/hatch.md b/docs/testing/hatch.md index ad756637d..f2cc518e6 100644 --- a/docs/testing/hatch.md +++ b/docs/testing/hatch.md @@ -38,10 +38,12 @@ environment plugin, source table or conditional override is refused before writing. Use agent mode (`scan --mode agent` + `socket-patch apply` in CI) for these shapes. Hosted PEP 735 groups are supported; vendored groups are refused because Hatch does not expand `{root:uri}` within dependency groups. Vendoring a member of a Hatch 1.16+ -workspace is refused the same way when a workspace environment declared in -an ancestor `pyproject.toml` or `hatch.toml` (up to the checkout root) -installs the package from the member's static `[project]` dependencies or -the optional dependencies of the member's `features`: Hatch reads those +workspace is refused the same way when a workspace environment, declared +in a `pyproject.toml` or `hatch.toml` of an ancestor or of another project +in the same checkout (a root may list a sibling through `../`), installs +the package from the member's static `[project]` dependencies, from the +optional dependencies of the member's `features`, or from an extra the +workspace names as `[extra]`: Hatch reads those strings raw, so the workspace environments would get an unexpanded `{root:uri}` (#505). Unknown direct sources require an explicit revert before patching. A hosted scan re-pins socket-patch's own From f4e25adba6474b521837c0a6b0ee3cb721f5adb7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 15:46:51 +0000 Subject: [PATCH 7/8] Read workspace extras the way Hatch installs them Collect the requirement strings that name the member from what the workspace environment actually installs: the root's dependencies and its environment features' extras, the environment's and its templates' dependencies (from hatch.toml when that replaces the pyproject tables), and the other members' dependencies and selected features. Resolve the extras they name through the same normalized lookup Hatch's resolve_extras uses, so a mixed-case extra name is found and strings Hatch never installs select nothing. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/utils/hatch_workspace.rs | 187 +++++++++++++----- 1 file changed, 137 insertions(+), 50 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index 8c743c59c..28910a0d2 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -63,8 +63,10 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option roots(&member, &lexical).into_iter().find_map(|root| { workspace_features(&root, &member, &own) .into_iter() - .find(|(_, features)| consumed(project, features).contains(&name)) - .map(|(env, _)| RawConsumer { root, env }) + .find(|(_, features, referenced)| { + consumed(project, features, referenced).contains(&name) + }) + .map(|(env, _, _)| RawConsumer { root, env }) }) } @@ -157,7 +159,11 @@ fn member_entry(table: &dyn TableLike) -> Option<(String, Vec)> { /// member's canonical name) the workspace's requirement strings name. /// Members are a string array, an array of inline tables or /// an array of tables (`[[...workspace.members]]`). -fn workspace_features(root: &Path, member: &Path, own: &str) -> Vec<(String, Vec)> { +fn workspace_features( + root: &Path, + member: &Path, + own: &str, +) -> Vec<(String, Vec, Vec)> { let external = parse(root, HATCH_FILES[1]); let project = parse(root, HATCH_FILES[0]); let (_, envs) = environments(project.as_ref(), external.as_ref()); @@ -197,62 +203,95 @@ fn workspace_features(root: &Path, member: &Path, own: &str) -> Vec<(String, Vec { continue; } - // `all_dependencies_complex` expands every member's raw extras - // that any of the environment's requirement strings name as - // `[extra]`: the root's own and its environments', - // and the other members' (#505 review). - let mut features = features.clone(); - let mut texts: Vec = project.iter().chain(&external).cloned().collect(); - texts.extend( - listed + // `all_dependencies_complex` runs `resolve_extras` over every + // requirement string the environment installs, with each + // member's raw optional dependencies: a `[extra]` among + // them installs that extra raw too (#505 review). Those strings + // are the root's dependencies and the optional dependencies of + // the environment's `features`, the environment's (and its + // templates') dependencies, and what the workspace takes from + // the other members. + let mut specs = environment_specs(envs, env); + if let Some(root_project) = project.as_ref().and_then(|doc| doc.get("project")) { + let env_features: Vec = environment_chain(envs, env) + .iter() + .flat_map(|config| strings(config.get("features"))) + .map(canonicalize_pypi_name) + .collect(); + specs.extend(selected_specs(root_project, &env_features)); + } + for (paths, features) in &listed { + for path in paths .iter() - .flat_map(|(paths, _)| paths) .filter(|path| real(path).as_deref() != Some(member)) - .filter_map(|path| parse(path, HATCH_FILES[0])), - ); - for document in &texts { - features.extend(referenced_extras(document.as_item(), own)); + { + if let Some(sibling) = parse(path, HATCH_FILES[0]) { + if let Some(sibling) = sibling.get("project") { + specs.extend(selected_specs(sibling, features)); + } + } + } } - found.push((env.to_string(), features)); + let referenced = specs + .into_iter() + .filter(|spec| canonicalize_pypi_name(pep508_name(spec)) == own) + .collect(); + found.push((env.to_string(), features.clone(), referenced)); } } found } -/// The normalized extras of `own` that any requirement string anywhere in -/// `item` names as `own[extra]`. -fn referenced_extras(item: &Item, own: &str) -> Vec { - fn walk(value: &Value, own: &str, out: &mut Vec) { - match value { - Value::String(text) => { - let spec = text.value(); - if canonicalize_pypi_name(pep508_name(spec)) == own { - out.extend(extras(spec)); - } - } - Value::Array(array) => array.iter().for_each(|value| walk(value, own, out)), - Value::InlineTable(table) => table.iter().for_each(|(_, value)| walk(value, own, out)), - _ => {} - } - } - let mut out = Vec::new(); - match item { - Item::Value(value) => walk(value, own, &mut out), - Item::Table(table) => { - for (_, item) in table.iter() { - out.extend(referenced_extras(item, own)); - } - } - Item::ArrayOfTables(tables) => { - for table in tables.iter() { - for (_, item) in table.iter() { - out.extend(referenced_extras(item, own)); - } - } +/// The environment named `env` followed by the templates it inherits +/// from (`template`, `default` unless set; an environment naming itself +/// ends the chain). +fn environment_chain<'e>(envs: &'e dyn TableLike, env: &str) -> Vec<&'e Item> { + let mut chain = Vec::new(); + let mut name = env.to_owned(); + let mut seen = BTreeSet::new(); + while seen.insert(name.clone()) { + let Some(config) = envs.get(&name) else { break }; + chain.push(config); + let next = config + .get("template") + .and_then(Item::as_str) + .unwrap_or("default"); + if next == name { + break; } - Item::None => {} + name = next.to_owned(); } - out + chain +} + +/// The `dependencies` and `extra-dependencies` of `env` and its templates. +fn environment_specs(envs: &dyn TableLike, env: &str) -> Vec { + environment_chain(envs, env) + .iter() + .flat_map(|config| { + ["dependencies", "extra-dependencies"] + .into_iter() + .flat_map(|key| strings(config.get(key))) + }) + .map(str::to_owned) + .collect() +} + +/// A `[project]` table's `dependencies` and the optional dependencies of +/// `features` (normalized names, looked up as exact keys). +fn selected_specs(project: &Item, features: &[String]) -> Vec { + let groups = project + .get("optional-dependencies") + .and_then(Item::as_table_like); + strings(project.get("dependencies")) + .into_iter() + .chain( + features.iter().flat_map(|feature| { + strings(groups.and_then(|groups| groups.get(feature.as_str()))) + }), + ) + .map(str::to_owned) + .collect() } /// A member spec with the context fields Hatch formats in it (`{root}`, @@ -426,7 +465,7 @@ fn tokenize(pattern: &[char]) -> Vec { /// table key exactly, and the member's references to its own extras /// (`lib[other]`) are followed through `resolve_extras`, which also /// accepts a normalized key. -fn consumed(project: &Item, features: &[String]) -> BTreeSet { +fn consumed(project: &Item, features: &[String], referenced: &[String]) -> BTreeSet { let own = project .get("name") .and_then(Item::as_str) @@ -449,6 +488,9 @@ fn consumed(project: &Item, features: &[String]) -> BTreeSet { let mut names = BTreeSet::new(); let mut seen = BTreeSet::new(); let mut queue: VecDeque<&str> = strings(project.get("dependencies")).into(); + // A workspace requirement naming the member (`lib1[Sec]`) reaches its + // extras through `resolve_extras`, which also takes a normalized key. + queue.extend(referenced.iter().map(String::as_str)); for feature in features { if seen.insert(feature.clone()) { queue.extend(groups.get(feature.as_str()).into_iter().flatten()); @@ -660,6 +702,51 @@ mod tests { assert_eq!(consumer(&dir, MEMBER, "certifi"), None); } + /// Bugbot: a workspace string naming a mixed-case extra reaches it + /// through `resolve_extras`'s normalized lookup; strings Hatch never + /// installs (environments a hatch.toml `envs` replaces, a sibling's + /// unselected extras) select nothing. + #[test] + fn referenced_extras_follow_what_the_environment_installs() { + let members = "[\"libs/*\"]"; + let env = format!( + "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[spaced.out]\"]\n\ + [tool.hatch.envs.default.workspace]\nmembers = {members}\n" + ); + let (_temp, dir) = layout(&env, None, "libs/lib1"); + assert_eq!( + consumer(&dir, MEMBER, "certifi").as_deref(), + Some("default") + ); + // A hatch.toml `envs` hides the pyproject environment naming it. + let shadowed = format!("{ROOT}[tool.hatch.envs.lint]\ndependencies = [\"lib1[sec]\"]\n"); + let (_temp, dir) = layout( + &shadowed, + Some("[envs.default.workspace]\nmembers = [\"libs/*\"]\n"), + "libs/lib1", + ); + assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); + assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); + // An environment inherits its template's dependencies. + let inherited = format!( + "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[sec]\"]\n\ + [tool.hatch.envs.test.workspace]\nmembers = {members}\n" + ); + let (_temp, dir) = layout(&inherited, None, "libs/lib1"); + assert_eq!(consumer(&dir, MEMBER, "urllib3").as_deref(), Some("test")); + // A sibling's extra the workspace does not select is not installed. + let (temp, dir) = layout(&workspace(members), None, "libs/lib1"); + let sibling = temp.path().join("ws/libs/lib2"); + std::fs::create_dir_all(&sibling).unwrap(); + std::fs::write( + sibling.join("pyproject.toml"), + "[project]\nname = \"lib2\"\nversion = \"1\"\n\ + [project.optional-dependencies]\nx = [\"lib1[sec]\"]\n", + ) + .unwrap(); + assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); + } + /// #505: a workspace declared in hatch.toml counts, and a hatch.toml /// `envs` table hides pyproject's environments. #[test] From 5a83ffc9ddce9266035093907df48c77f27cf2ec Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 16:06:25 +0000 Subject: [PATCH 8/8] Treat every extra of a Hatch workspace member as raw Which optional dependencies a Hatch workspace environment installs from a member depends on the whole workspace: the member's features, [extra] references from the root, environments, their templates and dependency groups, and extras of other members that reach it. Each review round found another path the precise model missed, and a miss writes a {root:uri} that breaks every workspace sync. Once a workspace lists the member, count all its static dependencies and optional dependencies as read raw instead. A refusal still leaves hosted and agent mode, and the module shrinks to member discovery. Assisted-by: Claude Code:claude-opus-5-5 --- .../src/utils/hatch_workspace.rs | 415 ++++-------------- .../src/vendor/pypi_hatch.rs | 22 +- docs/testing/hatch.md | 14 +- 3 files changed, 99 insertions(+), 352 deletions(-) diff --git a/crates/socket-patch-core/src/utils/hatch_workspace.rs b/crates/socket-patch-core/src/utils/hatch_workspace.rs index 28910a0d2..9068775c0 100644 --- a/crates/socket-patch-core/src/utils/hatch_workspace.rs +++ b/crates/socket-patch-core/src/utils/hatch_workspace.rs @@ -1,14 +1,20 @@ //! Hatch workspaces (Hatch 1.16+, #505): an environment's //! `workspace.members` installs each member editable together with the -//! member's `[project]` dependencies and the optional dependencies of the -//! member's `features`. For a member whose dependencies are static, Hatch -//! reads those strings from the member's `pyproject.toml` raw, without -//! context formatting, so a `{root:uri}` reference there reaches pip -//! unexpanded and every sync of the workspace environment fails. This -//! module finds the workspace environments that consume a project that -//! way, read the way Hatch 1.18's `Workspace.members` resolves them. +//! member's `[project]` dependencies, and its raw optional dependencies +//! whenever the member's `features` or any requirement the environment +//! installs names one of them (`lib[extra]`, expanded transitively across +//! the root and every member by `resolve_extras`). For a member whose +//! dependencies are static, Hatch reads those strings from the member's +//! `pyproject.toml` raw, without context formatting, so a `{root:uri}` +//! reference there reaches pip unexpanded and every sync of the workspace +//! environment fails. This module finds the workspace environments that +//! list a project, read the way Hatch 1.18's `Workspace.members` resolves +//! them. Which extras an environment ends up installing depends on the +//! whole workspace (environment templates, dependency groups, the root's +//! and every member's extras), so every optional dependency of a listed +//! member counts as read raw: a refusal leaves hosted and agent mode, a +//! missed one breaks the workspace. -use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::path::{Path, PathBuf}; use toml_edit::{DocumentMut, Item, TableLike, Value}; @@ -31,15 +37,15 @@ pub struct RawConsumer { pub env: String, } -/// The first workspace environment that installs `name` from the member's -/// static `[project]` dependencies, or from the optional dependencies of -/// the features it selects, read from `member_pyproject`. Workspace roots -/// are looked for in the member's ancestors and, when the member sits in a -/// git checkout, in every project of that checkout (a root can list a -/// sibling through a `../` spec); the walk never leaves the checkout. -/// `None` when no workspace reads `name` raw, including when the member's -/// dependencies are dynamic: Hatch then builds the metadata through the -/// backend, which expands `{root:uri}`. +/// The first workspace environment that lists the member and so may +/// install `name` raw from its static `[project]` dependencies or optional +/// dependencies, read from `member_pyproject`. Workspace roots are looked +/// for in the member's ancestors and, when the member sits in a git +/// checkout, in every project of that checkout (a root can list a sibling +/// through a `../` spec); the walk never leaves the checkout. `None` when +/// the member does not declare `name` there or no workspace lists it, +/// including when its dependencies are dynamic: Hatch then builds the +/// metadata through the backend, which expands `{root:uri}`. pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option { let document: DocumentMut = member_pyproject.parse().ok()?; let project = document.get("project")?; @@ -49,24 +55,31 @@ pub fn raw_consumer(member: &Path, member_pyproject: &str, name: &str) -> Option { return None; } + let name = canonicalize_pypi_name(name); + let declared = strings(project.get("dependencies")) + .into_iter() + .chain( + project + .get("optional-dependencies") + .and_then(Item::as_table_like) + .into_iter() + .flat_map(|groups| groups.iter()) + .flat_map(|(_, group)| strings(Some(group))), + ) + .any(|spec| canonicalize_pypi_name(pep508_name(spec)) == name); + if !declared { + return None; + } // Hatch reaches a member through symlinks without resolving them, so // roots are looked for above both spellings and found members are // compared by their real path. let lexical = absolute(member)?; let member = std::fs::canonicalize(&lexical).unwrap_or_else(|_| lexical.clone()); - let name = canonicalize_pypi_name(name); - let own = project - .get("name") - .and_then(Item::as_str) - .map(canonicalize_pypi_name) - .unwrap_or_default(); roots(&member, &lexical).into_iter().find_map(|root| { - workspace_features(&root, &member, &own) + listing_environments(&root, &member) .into_iter() - .find(|(_, features, referenced)| { - consumed(project, features, referenced).contains(&name) - }) - .map(|(env, _, _)| RawConsumer { root, env }) + .next() + .map(|env| RawConsumer { root, env }) }) } @@ -143,157 +156,51 @@ fn parse(dir: &Path, file: &str) -> Option { .ok() } -/// One `workspace.members` entry: its path spec and normalized features. -fn member_entry(table: &dyn TableLike) -> Option<(String, Vec)> { - let path = table.get("path")?.as_str()?.to_owned(); - let features = strings(table.get("features")) - .into_iter() - .map(canonicalize_pypi_name) - .collect(); - Some((path, features)) -} - -/// For each environment of the project at `root` whose workspace lists -/// `member`: the environment name and the member's selected features -/// (normalized): those the entry selects, plus every extra of `own` (the -/// member's canonical name) the workspace's requirement strings name. -/// Members are a string array, an array of inline tables or -/// an array of tables (`[[...workspace.members]]`). -fn workspace_features( - root: &Path, - member: &Path, - own: &str, -) -> Vec<(String, Vec, Vec)> { +/// The environments of the project at `root` whose workspace lists +/// `member` (a canonical path). Members are a string array, an array of +/// inline tables or an array of tables (`[[...workspace.members]]`), each +/// table naming its `path`. An environment that inherits a workspace from +/// its template lists the same members as the template, which is found. +fn listing_environments(root: &Path, member: &Path) -> Vec { let external = parse(root, HATCH_FILES[1]); let project = parse(root, HATCH_FILES[0]); let (_, envs) = environments(project.as_ref(), external.as_ref()); let Some(envs) = envs.and_then(Item::as_table_like) else { return Vec::new(); }; + let path_of = |table: &dyn TableLike| table.get("path")?.as_str().map(str::to_owned); let mut found = Vec::new(); for (env, config) in envs.iter() { let Some(workspace) = config.get("workspace") else { continue; }; let exclude = strings(workspace.get("exclude")); - let entries: Vec<(String, Vec)> = match workspace.get("members") { + let specs: Vec = match workspace.get("members") { Some(Item::Value(Value::Array(members))) => members .iter() .filter_map(|entry| match entry { - Value::String(path) => Some((path.value().clone(), Vec::new())), - Value::InlineTable(table) => member_entry(table), + Value::String(path) => Some(path.value().clone()), + Value::InlineTable(table) => path_of(table), _ => None, }) .collect(), - Some(Item::ArrayOfTables(members)) => members - .iter() - .filter_map(|table| member_entry(table)) - .collect(), + Some(Item::ArrayOfTables(members)) => { + members.iter().filter_map(|table| path_of(table)).collect() + } _ => continue, }; - let listed: Vec<(Vec, Vec)> = entries - .into_iter() - .map(|(path, features)| (members_of(root, &path, &exclude), features)) - .collect(); - let real = |path: &PathBuf| std::fs::canonicalize(path).ok(); - for (members, features) in &listed { - if !members + let lists = specs.iter().any(|spec| { + members_of(root, spec, &exclude) .iter() - .any(|path| real(path).as_deref() == Some(member)) - { - continue; - } - // `all_dependencies_complex` runs `resolve_extras` over every - // requirement string the environment installs, with each - // member's raw optional dependencies: a `[extra]` among - // them installs that extra raw too (#505 review). Those strings - // are the root's dependencies and the optional dependencies of - // the environment's `features`, the environment's (and its - // templates') dependencies, and what the workspace takes from - // the other members. - let mut specs = environment_specs(envs, env); - if let Some(root_project) = project.as_ref().and_then(|doc| doc.get("project")) { - let env_features: Vec = environment_chain(envs, env) - .iter() - .flat_map(|config| strings(config.get("features"))) - .map(canonicalize_pypi_name) - .collect(); - specs.extend(selected_specs(root_project, &env_features)); - } - for (paths, features) in &listed { - for path in paths - .iter() - .filter(|path| real(path).as_deref() != Some(member)) - { - if let Some(sibling) = parse(path, HATCH_FILES[0]) { - if let Some(sibling) = sibling.get("project") { - specs.extend(selected_specs(sibling, features)); - } - } - } - } - let referenced = specs - .into_iter() - .filter(|spec| canonicalize_pypi_name(pep508_name(spec)) == own) - .collect(); - found.push((env.to_string(), features.clone(), referenced)); + .any(|path| std::fs::canonicalize(path).ok().as_deref() == Some(member)) + }); + if lists { + found.push(env.to_string()); } } found } -/// The environment named `env` followed by the templates it inherits -/// from (`template`, `default` unless set; an environment naming itself -/// ends the chain). -fn environment_chain<'e>(envs: &'e dyn TableLike, env: &str) -> Vec<&'e Item> { - let mut chain = Vec::new(); - let mut name = env.to_owned(); - let mut seen = BTreeSet::new(); - while seen.insert(name.clone()) { - let Some(config) = envs.get(&name) else { break }; - chain.push(config); - let next = config - .get("template") - .and_then(Item::as_str) - .unwrap_or("default"); - if next == name { - break; - } - name = next.to_owned(); - } - chain -} - -/// The `dependencies` and `extra-dependencies` of `env` and its templates. -fn environment_specs(envs: &dyn TableLike, env: &str) -> Vec { - environment_chain(envs, env) - .iter() - .flat_map(|config| { - ["dependencies", "extra-dependencies"] - .into_iter() - .flat_map(|key| strings(config.get(key))) - }) - .map(str::to_owned) - .collect() -} - -/// A `[project]` table's `dependencies` and the optional dependencies of -/// `features` (normalized names, looked up as exact keys). -fn selected_specs(project: &Item, features: &[String]) -> Vec { - let groups = project - .get("optional-dependencies") - .and_then(Item::as_table_like); - strings(project.get("dependencies")) - .into_iter() - .chain( - features.iter().flat_map(|feature| { - strings(groups.and_then(|groups| groups.get(feature.as_str()))) - }), - ) - .map(str::to_owned) - .collect() -} - /// A member spec with the context fields Hatch formats in it (`{root}`, /// `{home}`) filled in; `None` when another field remains. fn expand(root: &Path, spec: &str) -> Option { @@ -459,73 +366,6 @@ fn tokenize(pattern: &[char]) -> Vec { tokens } -/// The canonical names of the dependencies a workspace installs from a -/// member's static `[project]` table: its `dependencies`, plus the optional -/// dependencies of `features`. As in Hatch, a selected feature names its -/// table key exactly, and the member's references to its own extras -/// (`lib[other]`) are followed through `resolve_extras`, which also -/// accepts a normalized key. -fn consumed(project: &Item, features: &[String], referenced: &[String]) -> BTreeSet { - let own = project - .get("name") - .and_then(Item::as_str) - .map(canonicalize_pypi_name); - let groups: BTreeMap<&str, Vec<&str>> = project - .get("optional-dependencies") - .and_then(Item::as_table_like) - .into_iter() - .flat_map(|table| table.iter()) - .map(|(key, group)| (key, strings(Some(group)))) - .collect(); - let normalized = |extra: &str| { - groups.get(extra).or_else(|| { - groups - .iter() - .find(|(key, _)| canonicalize_pypi_name(key) == extra) - .map(|(_, group)| group) - }) - }; - let mut names = BTreeSet::new(); - let mut seen = BTreeSet::new(); - let mut queue: VecDeque<&str> = strings(project.get("dependencies")).into(); - // A workspace requirement naming the member (`lib1[Sec]`) reaches its - // extras through `resolve_extras`, which also takes a normalized key. - queue.extend(referenced.iter().map(String::as_str)); - for feature in features { - if seen.insert(feature.clone()) { - queue.extend(groups.get(feature.as_str()).into_iter().flatten()); - } - } - while let Some(spec) = queue.pop_front() { - let name = canonicalize_pypi_name(pep508_name(spec)); - if own.as_deref() != Some(name.as_str()) { - names.insert(name); - continue; - } - for extra in extras(spec) { - if seen.insert(extra.clone()) { - queue.extend(normalized(&extra).into_iter().flatten()); - } - } - } - names -} - -/// The normalized extras of a PEP 508 spec (`name[a, b]`). -fn extras(spec: &str) -> Vec { - let rest = spec.trim_start()[pep508_name(spec).len()..].trim_start(); - rest.strip_prefix('[') - .and_then(|rest| rest.split_once(']')) - .map(|(list, _)| { - list.split(',') - .map(str::trim) - .filter(|extra| !extra.is_empty()) - .map(canonicalize_pypi_name) - .collect() - }) - .unwrap_or_default() -} - #[cfg(test)] mod tests { use super::*; @@ -629,122 +469,27 @@ mod tests { assert_eq!(consumer(&dir, MEMBER, "six"), None); } - /// #505: optional dependencies are consumed only through the member's - /// `features`, including extras the feature reaches through the - /// member's references to itself; a selected feature names its key - /// exactly. - #[test] - fn optional_dependencies_follow_member_features() { - let with = |features: &str| { - workspace(&format!( - "[{{ path = \"libs/lib1\", features = {features} }}]" - )) - }; - let (_temp, dir) = layout(&with("[\"sec\"]"), None, "libs/lib1"); - assert_eq!( - consumer(&dir, MEMBER, "urllib3").as_deref(), - Some("default") - ); - assert_eq!(consumer(&dir, MEMBER, "idna"), None); - let (_temp, dir) = layout(&with("[\"ALL\"]"), None, "libs/lib1"); - assert_eq!( - consumer(&dir, MEMBER, "urllib3").as_deref(), - Some("default") - ); - assert_eq!(consumer(&dir, MEMBER, "idna").as_deref(), Some("default")); - let (_temp, dir) = layout(&with("[]"), None, "libs/lib1"); - assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); - assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); - // Hatch looks the normalized `spaced-out` up as a raw key. - let (_temp, dir) = layout(&with("[\"Spaced_Out\"]"), None, "libs/lib1"); - assert_eq!(consumer(&dir, MEMBER, "certifi"), None); - } - - /// #505 review: an extra of the member that the workspace's own - /// requirement strings name (`lib1[sec]` in an environment, the root - /// project or another member) is installed raw too, transitively. + /// #505 and review: once a workspace lists the member, every optional + /// dependency counts as read raw, whatever reaches it (the entry's + /// `features`, a `lib1[sec]` in an environment, its template, a + /// dependency group, the root or another member, a mixed-case key); + /// a package the member does not declare is never refused. #[test] - fn extras_named_across_the_workspace_are_consumed() { - let members = "[\"libs/*\"]"; - let env = format!( - "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[sec]\"]\n\ - [tool.hatch.envs.default.workspace]\nmembers = {members}\n" - ); - let (_temp, dir) = layout(&env, None, "libs/lib1"); - assert_eq!( - consumer(&dir, MEMBER, "urllib3").as_deref(), - Some("default") - ); - assert_eq!(consumer(&dir, MEMBER, "idna"), None); - let root = ROOT.replace("version", "dependencies = [\"lib1[all]\"]\nversion"); - let (_temp, dir) = layout( - &format!("{root}[tool.hatch.envs.default.workspace]\nmembers = {members}\n"), - None, - "libs/lib1", - ); - assert_eq!( - consumer(&dir, MEMBER, "urllib3").as_deref(), - Some("default") - ); - assert_eq!(consumer(&dir, MEMBER, "idna").as_deref(), Some("default")); - let (temp, dir) = layout(&workspace(members), None, "libs/lib1"); - let sibling = temp.path().join("ws/libs/lib2"); - std::fs::create_dir_all(&sibling).unwrap(); - std::fs::write( - sibling.join("pyproject.toml"), - "[project]\nname = \"lib2\"\nversion = \"1\"\ndependencies = [\"LIB1[Sec]\"]\n", - ) - .unwrap(); - assert_eq!( - consumer(&dir, MEMBER, "urllib3").as_deref(), - Some("default") - ); - assert_eq!(consumer(&dir, MEMBER, "certifi"), None); - } - - /// Bugbot: a workspace string naming a mixed-case extra reaches it - /// through `resolve_extras`'s normalized lookup; strings Hatch never - /// installs (environments a hatch.toml `envs` replaces, a sibling's - /// unselected extras) select nothing. - #[test] - fn referenced_extras_follow_what_the_environment_installs() { - let members = "[\"libs/*\"]"; - let env = format!( - "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[spaced.out]\"]\n\ - [tool.hatch.envs.default.workspace]\nmembers = {members}\n" - ); - let (_temp, dir) = layout(&env, None, "libs/lib1"); - assert_eq!( - consumer(&dir, MEMBER, "certifi").as_deref(), - Some("default") - ); - // A hatch.toml `envs` hides the pyproject environment naming it. - let shadowed = format!("{ROOT}[tool.hatch.envs.lint]\ndependencies = [\"lib1[sec]\"]\n"); - let (_temp, dir) = layout( - &shadowed, - Some("[envs.default.workspace]\nmembers = [\"libs/*\"]\n"), - "libs/lib1", - ); - assert_eq!(consumer(&dir, MEMBER, "six").as_deref(), Some("default")); - assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); - // An environment inherits its template's dependencies. - let inherited = format!( - "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[sec]\"]\n\ - [tool.hatch.envs.test.workspace]\nmembers = {members}\n" - ); - let (_temp, dir) = layout(&inherited, None, "libs/lib1"); - assert_eq!(consumer(&dir, MEMBER, "urllib3").as_deref(), Some("test")); - // A sibling's extra the workspace does not select is not installed. - let (temp, dir) = layout(&workspace(members), None, "libs/lib1"); - let sibling = temp.path().join("ws/libs/lib2"); - std::fs::create_dir_all(&sibling).unwrap(); - std::fs::write( - sibling.join("pyproject.toml"), - "[project]\nname = \"lib2\"\nversion = \"1\"\n\ - [project.optional-dependencies]\nx = [\"lib1[sec]\"]\n", - ) - .unwrap(); - assert_eq!(consumer(&dir, MEMBER, "urllib3"), None); + fn every_optional_dependency_of_a_listed_member_counts() { + for root in [ + workspace("[\"libs/*\"]"), + workspace("[{ path = \"libs/lib1\", features = [\"sec\"] }]"), + format!( + "{ROOT}[tool.hatch.envs.default]\ndependencies = [\"lib1[Spaced_Out]\"]\n\ + [tool.hatch.envs.test.workspace]\nmembers = [\"libs/*\"]\n" + ), + ] { + let (_temp, dir) = layout(&root, None, "libs/lib1"); + for name in ["six", "urllib3", "idna", "certifi"] { + assert!(consumer(&dir, MEMBER, name).is_some(), "{root}: {name}"); + } + assert_eq!(consumer(&dir, MEMBER, "requests"), None, "{root}"); + } } /// #505: a workspace declared in hatch.toml counts, and a hatch.toml diff --git a/crates/socket-patch-core/src/vendor/pypi_hatch.rs b/crates/socket-patch-core/src/vendor/pypi_hatch.rs index 2b852be88..c800a168b 100644 --- a/crates/socket-patch-core/src/vendor/pypi_hatch.rs +++ b/crates/socket-patch-core/src/vendor/pypi_hatch.rs @@ -140,10 +140,10 @@ fn require_expandable_root(root: &Path) -> Result<(), Failure> { } } -/// Refuse a member of a Hatch workspace whose environment installs `name` -/// from this project's `[project]` table (#505): Hatch reads a member's -/// static dependencies raw, so the workspace root's environments would get -/// an unexpanded `{root:uri}` and fail every sync. +/// Refuse `name` in this project's `[project]` dependencies or optional +/// dependencies when a Hatch workspace lists the project (#505): Hatch +/// reads a member's static dependencies raw, so the workspace root's +/// environments would get an unexpanded `{root:uri}` and fail every sync. async fn require_no_raw_workspace( root: &Path, files: &BTreeMap, @@ -162,7 +162,7 @@ async fn require_no_raw_workspace( Some(consumer) => Err(( "pypi_hatch_unsupported", format!( - "this project is a member of the Hatch workspace at {} (environment {:?}), which installs {name} from this project's dependencies without expanding {{root:uri}}, so a vendored reference would break that workspace's environments; use hosted or agent mode{}", + "this project is a member of the Hatch workspace at {} (environment {:?}), which reads this project's dependencies and optional dependencies, {name} among them, without expanding {{root:uri}}, so a vendored reference would break that workspace's environments; use hosted or agent mode{}", consumer.root.display(), consumer.env, if vendored { @@ -554,10 +554,10 @@ mod tests { } } - /// #505: `load` and `preflight` refuse a member whose `[project]` - /// dependency (or a selected feature's) a Hatch workspace in an - /// ancestor pyproject.toml or hatch.toml installs raw, before writing; - /// the member's own environment dependencies stay vendorable. + /// #505: `load` and `preflight` refuse, before writing, a `[project]` + /// dependency or optional dependency of a member that a Hatch workspace + /// in an ancestor pyproject.toml or hatch.toml lists, since Hatch reads + /// those raw. #[tokio::test] async fn member_consumed_by_a_hatch_workspace_is_refused() { let member = @@ -569,7 +569,7 @@ mod tests { ( format!("{root}[tool.hatch.envs.default.workspace]\nmembers = [\"libs/*\"]\n"), None, - &["six"][..], + &["six", "one"][..], ), ( format!( @@ -582,7 +582,7 @@ mod tests { ( root.to_string(), Some("[envs.default.workspace]\nmembers = [\"libs/lib1\"]\n"), - &["six"][..], + &["six", "one"][..], ), ] { let temp = tempfile::tempdir().unwrap(); diff --git a/docs/testing/hatch.md b/docs/testing/hatch.md index f2cc518e6..6ac4e078f 100644 --- a/docs/testing/hatch.md +++ b/docs/testing/hatch.md @@ -40,12 +40,14 @@ supported; vendored groups are refused because Hatch does not expand `{root:uri}` within dependency groups. Vendoring a member of a Hatch 1.16+ workspace is refused the same way when a workspace environment, declared in a `pyproject.toml` or `hatch.toml` of an ancestor or of another project -in the same checkout (a root may list a sibling through `../`), installs -the package from the member's static `[project]` dependencies, from the -optional dependencies of the member's `features`, or from an extra the -workspace names as `[extra]`: Hatch reads those -strings raw, so the workspace environments would get an unexpanded -`{root:uri}` (#505). Unknown direct sources require an +in the same checkout (a root may list a sibling through `../`), lists the +member and the package is one of the member's static `[project]` +dependencies or optional dependencies. Hatch reads those strings raw, and +which extras it installs depends on the whole workspace (member +`features`, `[extra]` references from the root, environments, +templates, dependency groups and other members), so every optional +dependency of a listed member is treated as installed; hosted and agent +mode are unaffected (#505). Unknown direct sources require an explicit revert before patching. A hosted scan re-pins socket-patch's own earlier hosted reference (same package and version on the patch server) when the grant rotates or a newer patch supersedes it. A vendored scan