diff --git a/crates/socket-patch-cli/tests/mode_migration_pypi.rs b/crates/socket-patch-cli/tests/mode_migration_pypi.rs index 901ea8255..5535a27d8 100644 --- a/crates/socket-patch-cli/tests/mode_migration_pypi.rs +++ b/crates/socket-patch-cli/tests/mode_migration_pypi.rs @@ -3289,6 +3289,69 @@ async fn pip_lock_vendor_wires_requirements_with_the_pylock() { assert!(!root.join(format!(".socket/vendor/pypi/{UUID}")).exists()); } +/// #1851: requirements.txt pins six only through a `-r` include outside +/// the project root, beside the `pip lock` pylock.toml. `pip install -r +/// requirements.txt` installs that pin, and vendor can't edit the +/// include, so vendoring six refuses with `pypi_requirements_outside_root` +/// (as it does without the pylock) and writes nothing. Before the fix only +/// the pylock was wired and the run exited 0. A pylock already wired +/// beside such a pin (vendored before the fix, or a pin added later) +/// fails `vendor --check` as contested instead of passing. +#[tokio::test] +async fn pip_lock_beside_an_out_of_root_include_pin_refuses() { + let (tmp, root) = project(); + stage_pip_lock(&root); + let shared = tmp.path().join("shared"); + std::fs::create_dir_all(&shared).unwrap(); + std::fs::write(shared.join("base.txt"), "six==1.16.0\n").unwrap(); + std::fs::write( + root.join("requirements.txt"), + "-r ../shared/base.txt\nidna==3.7\n", + ) + .unwrap(); + stage_manifest(&root); + + let (code, env) = run_cli(&root, &["vendor"], &[]); + assert_ne!(code, 0, "{env:#}"); + assert!( + env.to_string().contains("pypi_requirements_outside_root"), + "{env:#}" + ); + assert_eq!( + std::fs::read(root.join("pylock.toml")).unwrap(), + PIP_LOCK.as_bytes(), + "the pylock is not wired: {env:#}" + ); + assert_eq!( + std::fs::read(root.join("requirements.txt")).unwrap(), + b"-r ../shared/base.txt\nidna==3.7\n" + ); + assert_eq!( + std::fs::read(shared.join("base.txt")).unwrap(), + b"six==1.16.0\n" + ); + + // The include pins another release: six is wired in the pylock only. + std::fs::write(shared.join("base.txt"), "six==1.15.0\n").unwrap(); + let (code, env) = run_cli(&root, &["vendor"], &[]); + assert_eq!(code, 0, "{env:#}"); + assert!( + std::fs::read_to_string(root.join("pylock.toml")) + .unwrap() + .contains(&format!(".socket/vendor/pypi/{UUID}/")), + "{env:#}" + ); + let (code, env) = run_cli(&root, &["vendor", "--check"], &[]); + assert_eq!(code, 0, "{env:#}"); + + // Once the include pins the vendored release, pip installs the + // upstream six again: the pylock wiring is contested. + std::fs::write(shared.join("base.txt"), "six==1.16.0\n").unwrap(); + let (code, env) = run_cli(&root, &["vendor", "--check"], &[]); + assert_ne!(code, 0, "the out-of-root pin contests the pylock: {env:#}"); + assert!(env.to_string().contains("contested"), "{env:#}"); +} + /// #1758: a pylock vendored while requirements.txt did not pin six yet /// (a project vendored before the fix wired only the pylock). Once the /// file pins six==1.16.0, the next `vendor` wires it to the committed diff --git a/crates/socket-patch-core/src/utils/requirements.rs b/crates/socket-patch-core/src/utils/requirements.rs index cc7384db3..cb1851497 100644 --- a/crates/socket-patch-core/src/utils/requirements.rs +++ b/crates/socket-patch-core/src/utils/requirements.rs @@ -381,6 +381,40 @@ pub(crate) fn shlex_split(text: &str) -> Option> { Some(words) } +/// The location of an editable requirement (`-e `, `-e`, +/// `--editable `, `--editable=`), as pip's option parser reads it. +pub(crate) fn editable_location(code: &str) -> Option<&str> { + let rest = if let Some(rest) = code.strip_prefix("--editable") { + match rest.strip_prefix('=') { + Some(rest) => rest, + None if rest.starts_with(char::is_whitespace) => rest, + None => return None, + } + } else { + // A short option takes its value attached or after a space. + let rest = code.strip_prefix("-e")?; + rest.strip_prefix('=').unwrap_or(rest) + }; + rest.split_whitespace().next() +} + +/// The package a VCS / url requirement names in its `#egg=` fragment. +/// A legacy `#egg=-` fragment names the package before its +/// first `-`, as setuptools and older pip read it. +pub(crate) fn egg_name(location: &str) -> Option<&str> { + let (_, fragment) = location.split_once('#')?; + let egg = fragment.split('&').find_map(|p| p.strip_prefix("egg="))?; + let end = egg + .find(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))) + .unwrap_or(egg.len()); + let egg = &egg[..end]; + let end = egg + .match_indices('-') + .find(|(i, _)| egg[i + 1..].starts_with(|c: char| c.is_ascii_digit())) + .map_or(egg.len(), |(i, _)| i); + Some(&egg[..end]).filter(|name| !name.is_empty()) +} + /// One requirements.txt requirement, read from a logical line's code part /// (comment already stripped) the way pip reads it. The ONE requirements /// grammar: the lock inventory, lockfile discovery, the hosted rewriter and diff --git a/crates/socket-patch-core/src/vendor/pypi.rs b/crates/socket-patch-core/src/vendor/pypi.rs index 34dd35d13..c52d494cc 100644 --- a/crates/socket-patch-core/src/vendor/pypi.rs +++ b/crates/socket-patch-core/src/vendor/pypi.rs @@ -244,6 +244,25 @@ async fn requirements_pins_target(project_root: &Path, target: Option<(&str, &st } } +/// #1851: refuse wiring a lock beside a requirements set whose exact pin of +/// the target sits in an out-of-root `-r` include. pip installs that pin, +/// vendor cannot edit the file, and naming requirements.txt a loser would +/// let the lock wiring report the package patched. +/// A pin the lock wiring rewrites with it ([`super::pypi_requirements:: +/// sibling_pin_wirable`]) never has one: the planner refuses it first. +async fn refuse_pin_outside_root( + project_root: &Path, + target: Option<(&str, &str)>, +) -> Result<(), (&'static str, String)> { + let Some((name, version)) = target else { + return Ok(()); + }; + match super::pypi_requirements::pin_outside_root(project_root, name, version).await { + Some(refusal) => Err(refusal), + None => Ok(()), + } +} + /// Route the project to a wiring flavor, first match wins. Lockfiles are the /// authoritative "this tool manages installs" signal, so locks are compared /// with locks (precedence follows migration direction / ecosystem currency: @@ -396,6 +415,7 @@ async fn detect_pypi_flavor( } None => false, }; + refuse_pin_outside_root(project_root, target).await?; if !wired_with_lock && exists(PYPI_REQUIREMENTS).await { present.push(PYPI_REQUIREMENTS); } @@ -449,6 +469,9 @@ async fn detect_pypi_flavor( } _ => false, }; + if governing.is_some() { + refuse_pin_outside_root(project_root, target).await?; + } if governing.is_some() && !wired_with_lock && requirements_pins_target(project_root, target).await @@ -4138,6 +4161,132 @@ mod tests { } } + /// #1851: pip installs an exact pin from a `-r` include outside the + /// project root, which vendor cannot edit. Wiring only the lock beside + /// it would leave `pip install -r requirements.txt` on the upstream + /// release, so every lock flavor refuses it as the requirements-only + /// planner does, instead of naming requirements.txt a loser. + #[tokio::test] + async fn exact_pin_in_an_out_of_root_include_refuses_beside_a_lock() { + let pylock = PIPENV_PYLOCK.replace("pipenv", "pip"); + for (lock, content) in [ + ("pylock.toml", pylock.as_str()), + ("uv.lock", "version = 1\n"), + ("Pipfile.lock", "{}"), + ("poetry.lock", ""), + ] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("app"); + tokio::fs::create_dir_all(&root).await.unwrap(); + touch(tmp.path(), "base.txt", "six==1.16.0\n").await; + touch(&root, lock, content).await; + touch(&root, "requirements.txt", "-r ../base.txt\nidna==3.7\n").await; + let (code, detail) = detect_pypi_flavor(&root, Some(("six", "1.16.0"))) + .await + .unwrap_err(); + assert_eq!(code, "pypi_requirements_outside_root", "{lock}: {detail}"); + assert!(detail.contains("../base.txt"), "{lock}: {detail}"); + + // Another release, or a package only the in-root file names, + // is not refused. + touch(tmp.path(), "base.txt", "six==1.15.0\n").await; + assert!( + detect_pypi_flavor(&root, Some(("six", "1.16.0"))) + .await + .is_ok(), + "{lock}" + ); + assert!( + detect_pypi_flavor(&root, Some(("idna", "3.7"))) + .await + .is_ok(), + "{lock}" + ); + } + } + + /// #1851: every shape through which pip installs the release from an + /// out-of-root include refuses, as discovery would contest the wiring + /// it leaves: extras, `===`, per-requirement options, a direct + /// reference, an editable checkout, and a UTF-16 `pip freeze` export. + #[tokio::test] + async fn every_installing_shape_in_an_out_of_root_include_refuses() { + let pylock = PIPENV_PYLOCK.replace("pipenv", "pip"); + let utf16: Vec = [0xFF, 0xFE] + .into_iter() + .chain("six==1.16.0\r\n".encode_utf16().flat_map(u16::to_le_bytes)) + .collect(); + let mut shapes: Vec<(&str, Vec)> = [ + "six[socks]==1.16.0\n", + "six===1.16.0\n", + "Six==1.16.0 --config-settings=x=y\n", + "six @ https://files.pythonhosted.org/packages/six-1.16.0-py2.py3-none-any.whl\n", + "-e git+https://example.com/six.git#egg=six\n", + "--editable=git+https://example.com/six.git#egg=six\n", + "git+https://example.com/six.git#egg=six\n", + "https://files.pythonhosted.org/packages/six-1.16.0-py2.py3-none-any.whl\n", + "../wheels/six-1.16.0-py2.py3-none-any.whl\n", + ] + .into_iter() + .map(|text| (text, text.as_bytes().to_vec())) + .collect(); + shapes.push(("utf-16 six==1.16.0", utf16)); + for (shape, bytes) in shapes { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("app"); + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(tmp.path().join("base.txt"), &bytes) + .await + .unwrap(); + touch(&root, "pylock.toml", &pylock).await; + touch(&root, "requirements.txt", "-r ../base.txt\n").await; + let err = detect_pypi_flavor(&root, Some(("six", "1.16.0"))).await; + assert_eq!( + err.map_err(|(code, _)| code).map(|(flavor, _)| flavor), + Err("pypi_requirements_outside_root"), + "{shape}" + ); + } + + // An include pip may read but this walk can't decode: its pins are + // unknowable, so it refuses too. + { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("app"); + tokio::fs::create_dir_all(&root).await.unwrap(); + tokio::fs::write(tmp.path().join("base.txt"), b"\x80\x81six==1.16.0\n") + .await + .unwrap(); + touch(&root, "pylock.toml", &pylock).await; + touch(&root, "requirements.txt", "-r ../base.txt\n").await; + let (code, detail) = detect_pypi_flavor(&root, Some(("six", "1.16.0"))) + .await + .unwrap_err(); + assert_eq!(code, "pypi_requirements_outside_root", "{detail}"); + assert!(detail.contains("cannot be read"), "{detail}"); + } + + // A bare artifact of another release, or of another package, is + // not this pin. + for other in [ + "https://files.pythonhosted.org/packages/six-1.15.0-py2.py3-none-any.whl\n", + "../wheels/idna-3.7-py3-none-any.whl\n", + ] { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("app"); + tokio::fs::create_dir_all(&root).await.unwrap(); + touch(tmp.path(), "base.txt", other).await; + touch(&root, "pylock.toml", &pylock).await; + touch(&root, "requirements.txt", "-r ../base.txt\n").await; + assert!( + detect_pypi_flavor(&root, Some(("six", "1.16.0"))) + .await + .is_ok(), + "{other}" + ); + } + } + /// A pylock `pipenv lock` writes with `[pipenv] use_pylock = true`. const PIPENV_PYLOCK: &str = "lock-version = \"1.0\"\ncreated-by = \"pipenv\"\n\n[[packages]]\nname = \"six\"\nversion = \"1.16.0\"\nindex = \"https://pypi.org/simple\"\nwheels = [{ name = \"six-1.16.0-py2.py3-none-any.whl\", url = \"https://files.pythonhosted.org/packages/six-1.16.0-py2.py3-none-any.whl\", hashes = { sha256 = \"8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254\" } }]\n\n[tool.pipenv]\ngenerated_from = \"Pipfile.lock\"\n"; diff --git a/crates/socket-patch-core/src/vendor/pypi_requirements.rs b/crates/socket-patch-core/src/vendor/pypi_requirements.rs index db1a76d23..eacda979e 100644 --- a/crates/socket-patch-core/src/vendor/pypi_requirements.rs +++ b/crates/socket-patch-core/src/vendor/pypi_requirements.rs @@ -465,6 +465,105 @@ pub(super) async fn sibling_pin_wirable(root: &Path, canon_name: &str, version: && refuse_symlinked(root, &rels).await.is_ok() } +/// The out-of-root `-r` include (`../shared/base.txt`) through which pip +/// installs `canon_name==version` from somewhere other than a vendored +/// wheel, if the requirements set has one (#1851). Vendor cannot edit that +/// file, so wiring only a lock beside it would leave `pip install -r` on +/// the upstream release: the lock flavors refuse it as the +/// requirements-only planner does (`pypi_requirements_outside_root`). +/// +/// Files are decoded as pip decodes them (a UTF-16 `pip freeze` export +/// included), and a line counts when discovery would read it as evidence +/// against the wiring: an `==` / `===` pin of the release (extras and +/// per-requirement options allowed), a direct reference or editable +/// checkout naming the package (any version: its artifact is not ours), +/// and a bare url / VCS / path line whose artifact filename or `#egg=` +/// names it (at the filename's version, else any). An out-of-root include +/// that exists but can't be read or decoded refuses too: its pins are +/// unknowable. `Some` is the refusal to return. +pub(super) async fn pin_outside_root( + root: &Path, + canon_name: &str, + version: &str, +) -> Option<(&'static str, String)> { + let view = crate::vendor::lock_inventory::ProjectView::Disk(root); + let mut found = None; + let _ = walk_requirements_tree(view, decode_requirements, |rel, read| { + if found.is_some() { + return Ok::(false); + } + let outside = !is_in_root_rel(rel); + match read { + Ok(content) => { + if outside && installs_outside_vendor(&content, canon_name, version) { + found = Some(outside_root_refusal(rel, canon_name)); + } + Ok(true) + } + // pip reads an include this walk can't (or fails the install): + // its pins are unknowable, so fail closed rather than wire the + // lock beside it. Discovery diagnoses the same file. + Err(e) if outside && e.kind() != std::io::ErrorKind::NotFound => { + found = Some(( + "pypi_requirements_outside_root", + format!( + "{rel}: a requirements include outside the project root cannot be \ + read ({e}), so vendor cannot tell whether pip installs {canon_name} \ + from it; make it readable UTF-8 text or use agent mode (`scan --mode \ + agent` + `socket-patch apply`) instead" + ), + )); + Ok(false) + } + Err(_) => Ok(false), + } + }) + .await; + found +} + +/// Whether a requirements file's `content` installs `canon_name==version` +/// (see [`pin_outside_root`]) from a source other than a vendored wheel. +fn installs_outside_vendor(content: &str, canon_name: &str, version: &str) -> bool { + use crate::utils::requirements::{ + editable_location, egg_name, logical_lines, parse_requirement, split_comment, + }; + let names = |name: &str| canonicalize_pypi_name(name) == canon_name; + logical_lines(content).iter().any(|line| { + let code = split_comment(&line.text).0.trim(); + if let Some(location) = editable_location(code) { + return egg_name(location).is_some_and(names); + } + if let Some((name, pinned)) = crate::vex::discover::pypi_other::bare_line_release(code) { + return names(&name) + && pinned + .is_none_or(|pinned| crate::utils::pep440::versions_equal(&pinned, version)); + } + let Some(req) = parse_requirement(code).filter(|req| names(req.name)) else { + return false; + }; + req.direct_reference + .is_some_and(|location| !location.replace('\\', "/").contains(".socket/vendor/")) + || req + .exact_version() + .is_some_and(|pinned| crate::utils::pep440::versions_equal(pinned, version)) + || req.arbitrary_version() == Some(version) + }) +} + +/// The `pypi_requirements_outside_root` refusal for a pin in the +/// out-of-root include `rel`. +pub(super) fn outside_root_refusal(rel: &str, canon_name: &str) -> (&'static str, String) { + ( + "pypi_requirements_outside_root", + format!( + "{rel}: {canon_name} is pinned in a requirements include outside the project \ + root, which vendor cannot edit; inline it or use agent mode \ + (`scan --mode agent` + `socket-patch apply`) instead" + ), + ) +} + /// The refusal for a requirements pin with extras, which a vendored wheel /// path line cannot express. fn extras_refusal(rel: &str, canon_name: &str) -> (&'static str, String) { @@ -961,15 +1060,7 @@ async fn plan_requirements( // SECURITY/scope: an include outside the project root cannot be // edited by a committable vendor flow; rewriting only the in-root // copy would leave pip a duplicate requirement. Fail closed. - return Err(( - "pypi_requirements_outside_root", - format!( - "{}: {canon_name} is pinned in a requirements include outside the project \ - root, which vendor cannot edit; inline it or use agent mode \ - (`scan --mode agent` + `socket-patch apply`) instead", - file.rel - ), - )); + return Err(outside_root_refusal(&file.rel, canon_name)); } // Rewrite EVERY exact-pin occurrence in this file, bottom-up so the @@ -1315,6 +1406,30 @@ pub(crate) async fn requirements_include_names_in( Ok(names) } +/// The out-of-root (`../…`) and absolute `-r` includes the root +/// `requirements.txt` reaches, through in-root files and through each +/// other, in walk order (#1851). Vendor never edits or names them +/// ([`requirements_include_names_in`]), but pip installs their pins, so +/// discovery reads them as evidence against a lock's wiring. A file that +/// exists but can't be read or decoded is named too, so the reader +/// diagnoses it instead of losing its pins silently. +pub(crate) async fn requirements_outside_root_names_in( + view: crate::vendor::lock_inventory::ProjectView<'_>, +) -> Vec { + let mut names: Vec = Vec::new(); + let _ = walk_requirements_tree(view, decode_requirements, |rel, read| { + // An unreadable one is still named, so the reader diagnoses it. + if !is_in_root_rel(rel) + && !matches!(&read, Err(e) if e.kind() == std::io::ErrorKind::NotFound) + { + names.push(rel.to_string()); + } + Ok::(read.is_ok()) + }) + .await; + names +} + /// A root-relative requirements path that stays inside the project root /// (not `../…`, not absolute) — the only files the planner may edit. pub(crate) fn is_in_root_rel(rel: &str) -> bool { diff --git a/crates/socket-patch-core/src/vex/discover/pypi_locks.rs b/crates/socket-patch-core/src/vex/discover/pypi_locks.rs index c353a44cc..8e7a78166 100644 --- a/crates/socket-patch-core/src/vex/discover/pypi_locks.rs +++ b/crates/socket-patch-core/src/vex/discover/pypi_locks.rs @@ -1392,6 +1392,80 @@ mod tests { ) } + /// #1851: `pip install -r requirements.txt` installs an exact pin from a + /// `-r` include outside the project root, so that pin contests a + /// vendored pylock's wiring of the same release (`vendor --check` fails, + /// `vex` omits it), as an in-root pin does. Another release in the + /// include, or a package the include doesn't pin, leaves the wiring live. + #[tokio::test] + async fn out_of_root_include_pin_contests_a_vendored_pylock() { + let rel = click_vendored(UUID_B); + let lock = rewrite_python_lock( + &pylock_registry(), + "click", + "8.1.7", + ArtifactSource::Path(&rel), + SHA, + ) + .unwrap() + .unwrap(); + for (base, contested) in [ + ("click==8.1.7\n", true), + ("-r more.txt\n", true), + ("click[extra]===8.1.7 --config-settings=x=y\n", true), + ( + "click @ https://files.pythonhosted.org/packages/click-8.1.7-py3-none-any.whl\n", + true, + ), + ("-e git+https://example.com/click.git#egg=click\n", true), + ("git+https://example.com/click.git#egg=click\n", true), + ("../wheels/click-8.1.7-py3-none-any.whl\n", true), + ( + "click @ https://files.pythonhosted.org/packages/click-8.1.6-py3-none-any.whl\n", + false, + ), + ("click==8.1.6\n", false), + ("idna==3.7\n", false), + ] { + let p = Project::nested("app"); + p.write("pylock.toml", &lock) + .write("requirements.txt", "-r ../shared/base.txt\nidna==3.7\n") + .write("../shared/base.txt", base) + .write("../shared/more.txt", "click==8.1.7\n"); + // The whole project: the requirements extractor supplies the pin. + let out = p.discover().await; + if contested { + assert_refs(&out, &[]); + assert_eq!(out.contested.len(), 1, "{base:?}: {:#?}", out.contested); + assert_eq!(out.contested[0].file, Path::new("pylock.toml"), "{base:?}"); + } else { + assert_refs(&out, &[(CLICK, UUID_B, WiringMode::Vendored)]); + assert!(out.contested.is_empty(), "{base:?}: {:#?}", out.contested); + } + } + + // An include pip can't decode for this reader is diagnosed, not + // silently dropped. + let p = Project::nested("app"); + p.write("pylock.toml", &lock) + .write("requirements.txt", "-r ../shared/base.txt\n"); + std::fs::create_dir_all(p.root().join("../shared")).unwrap(); + std::fs::write( + p.root().join("../shared/base.txt"), + b"\x80\x81click==8.1.7\n", + ) + .unwrap(); + let out = p.discover().await; + assert!( + out.diagnostics + .iter() + .any(|d| d.code == DIAG_LOCKFILE_UNREADABLE + && d.detail.contains("../shared/base.txt")), + "{:#?}", + out.diagnostics + ); + } + #[tokio::test] async fn pylock_archive_hosted_and_vendored() { let url = click_url(UUID_A); diff --git a/crates/socket-patch-core/src/vex/discover/pypi_other.rs b/crates/socket-patch-core/src/vex/discover/pypi_other.rs index c6f55e57f..376b38626 100644 --- a/crates/socket-patch-core/src/vex/discover/pypi_other.rs +++ b/crates/socket-patch-core/src/vex/discover/pypi_other.rs @@ -66,7 +66,11 @@ //! and a stale one would otherwise gate the live wiring as a //! `wiring_conflict`. Lines are pip's logical lines, cut with the planner's //! own lexer ([`crate::utils::requirements`]: `\` continuations joined, `#` -//! comments at column 0 or after whitespace, one leading BOM): +//! comments at column 0 or after whitespace, one leading BOM). +//! Includes outside the root (`-r ../shared/base.txt`) are read too, but +//! only as evidence: pip installs their registry pins, which contest a +//! lock's wiring of the same release (#1851); neither writer edits them, +//! so nothing in them is wiring. The wiring shapes are: //! //! * hosted (`patch::redirect::requirements`): //! `Name[extras] @ [ ; marker] --hash=sha256:`; @@ -119,7 +123,8 @@ use crate::utils::digest::sha256_hex; use crate::utils::hatch::dependency_specs; use crate::utils::requirements; use crate::utils::requirements::{ - hash_options, logical_lines, parse_requirement, split_comment, url_sha256_fragment, vendor_tag, + editable_location, egg_name, hash_options, logical_lines, parse_requirement, split_comment, + url_sha256_fragment, vendor_tag, }; use crate::vendor::common::DeclTable; use crate::vendor::lock_inventory::pypi::{hosted_artifact_url, parse_pipfile_lock}; @@ -264,6 +269,56 @@ async fn extract_requirements(ctx: &DiscoverCtx<'_>, out: &mut Discovery) { requirements_line(ctx, file, &line.text, out); } } + // #1851: pip installs the pins of an include outside the project root + // too. Neither writer edits such a file, so it holds no Socket wiring, + // but its registry pins contest a lock's wiring like an in-root pin. + let budget = MAX_REQUIREMENTS_FILES.saturating_sub(files.len()); + let outside = + crate::vendor::pypi_requirements::requirements_outside_root_names_in(ctx.view).await; + for file in outside.iter().take(budget) { + out.install_tree(file, ROOT_REQUIREMENTS); + let Some(text) = ctx.read_requirements_text(file, out).await else { + continue; + }; + for line in logical_lines(&text) { + outside_root_line(ctx, file, &line.text, out); + } + } +} + +/// One logical line of an out-of-root include. Nothing in it is wiring, +/// but what pip installs from it is evidence: a registry pin, an editable +/// checkout, and a direct reference to anything but a Socket artifact +/// (read as [`foreign_reference`] reads an in-root one). +fn outside_root_line(ctx: &DiscoverCtx<'_>, file: &str, line: &str, out: &mut Discovery) { + let (code, _) = split_comment(line); + let code = code.trim(); + if let Some(location) = editable_location(code) { + if !mentions_vendor_dir(location) { + if let Some(name) = egg_name(location) { + out.resolved_elsewhere_any_version(file, pypi_name_purl(name)); + } + } + return; + } + if code.is_empty() || code.starts_with('-') { + return; + } + let Some((name, location)) = requirement_location(code) else { + registry_pin(file, code, out); + return; + }; + if matches!(classify(ctx, location, url_form(location)), Ok(None)) { + let entry = Entry { + file, + name, + pinned_version: None, + tag: None, + root_uri: false, + foreign_is_evidence: true, + }; + foreign_reference(&entry, location, out); + } } /// One logical requirements line (see the module docs for the two shapes). @@ -309,40 +364,6 @@ fn requirements_line(ctx: &DiscoverCtx<'_>, file: &str, line: &str, out: &mut Di ); } -/// The location of an editable requirement (`-e `, `-e`, -/// `--editable `, `--editable=`), as pip's option parser reads it. -fn editable_location(code: &str) -> Option<&str> { - let rest = if let Some(rest) = code.strip_prefix("--editable") { - match rest.strip_prefix('=') { - Some(rest) => rest, - None if rest.starts_with(char::is_whitespace) => rest, - None => return None, - } - } else { - // A short option takes its value attached or after a space. - let rest = code.strip_prefix("-e")?; - rest.strip_prefix('=').unwrap_or(rest) - }; - rest.split_whitespace().next() -} - -/// The package a VCS / url requirement names in its `#egg=` fragment. -/// A legacy `#egg=-` fragment names the package before its -/// first `-`, as setuptools and older pip read it. -fn egg_name(location: &str) -> Option<&str> { - let (_, fragment) = location.split_once('#')?; - let egg = fragment.split('&').find_map(|p| p.strip_prefix("egg="))?; - let end = egg - .find(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))) - .unwrap_or(egg.len()); - let egg = &egg[..end]; - let end = egg - .match_indices('-') - .find(|(i, _)| egg[i + 1..].starts_with(|c: char| c.is_ascii_digit())) - .map_or(egg.len(), |(i, _)| i); - Some(&egg[..end]).filter(|name| !name.is_empty()) -} - /// Record a registry requirement that can only install one release as /// evidence against another lock's wiring of it: an exact /// `name[extras]==X` pin in any PEP 440 spelling (`==1.16.0.0`, @@ -587,29 +608,48 @@ fn emit( /// requirements line names its package only through its filename or an /// `#egg=` fragment, so one with neither (a source directory) is left out. fn foreign_reference(entry: &Entry<'_>, location: &str, out: &mut Discovery) { + let Some((name, version)) = foreign_release(entry.name, location, entry.pinned_version) else { + return; + }; + match version { + Some(version) => out.resolved_elsewhere(entry.file, pypi_purl(&name, &version)), + None => out.resolved_elsewhere_any_version(entry.file, pypi_name_purl(&name)), + } +} + +/// The `(name, version)` a non-Socket reference `location` installs, read +/// as [`foreign_reference`] documents: `name` is the declared one, else the +/// artifact filename's, else the `#egg=` fragment's; `None` version means +/// any. `None` when no package name can be read. +fn foreign_release<'a>( + declared: Option<&'a str>, + location: &'a str, + pinned: Option<&'a str>, +) -> Option<(String, Option)> { let leaf = artifact_leaf(location).and_then(|leaf| vendored_leaf_purl("pypi", &leaf)); let leaf = leaf .as_deref() .and_then(|purl| purl.strip_prefix("pkg:pypi/")) .and_then(|nv| nv.rsplit_once('@')); - let Some(name) = entry - .name + let name = declared .or(leaf.map(|(name, _)| name)) - .or_else(|| egg_name(location)) - else { - return; - }; + .or_else(|| egg_name(location))?; let version = leaf .filter(|(leaf_name, _)| *leaf_name == canonicalize_pypi_name(name)) .map(|(_, version)| version) - .or_else(|| { - entry - .pinned_version - .filter(|v| !v.is_empty() && !v.contains('*')) - }); - match version { - Some(version) => out.resolved_elsewhere(entry.file, pypi_purl(name, version)), - None => out.resolved_elsewhere_any_version(entry.file, pypi_name_purl(name)), + .or_else(|| pinned.filter(|v| !v.is_empty() && !v.contains('*'))); + Some((name.to_string(), version.map(str::to_string))) +} + +/// The `(name, version)` a bare url / path requirements line +/// (`git+https://…#egg=six`, `../six-1.16.0-py3-none-any.whl`) installs, as +/// discovery reads it ([`foreign_release`]; `None` version means any). +/// `None` for any other line, and for a vendored wheel path. The vendored +/// planner refuses the same lines discovery counts as evidence (#1851). +pub(crate) fn bare_line_release(code: &str) -> Option<(String, Option)> { + match requirement_location(code)? { + (None, location) if !mentions_vendor_dir(location) => foreign_release(None, location, None), + _ => None, } }