diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 77553c64fa..a55fdb541f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -812,6 +812,9 @@ jobs: # same failure. Both are only observable in a `pull_request` run, so they # are covered by unit tests instead. - run: uv run --no-project --with pytest pytest scripts/test_check_ecosystem_roundtrip.py + # The build backend decides what the staged `pyproject.toml` says and where + # a dynamic version comes from. Neither is reachable from the Rust tests. + - run: uv run --no-project --with pytest pytest scripts/test_build_backend.py # Lint/format/type-check py-fuzzer # (dogfooding with ty is done in a separate job) - run: uv run --directory=./python/py-fuzzer mypy diff --git a/Cargo.lock b/Cargo.lock index 7232328ef5..bd24958b0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4735,6 +4735,7 @@ dependencies = [ "ty_python_core", "ty_python_semantic", "ty_server", + "ty_site_packages", "ty_static", "walkdir", "wild", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index f834b2b9da..d0dfde8438 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -3528,6 +3528,7 @@ dependencies = [ "rayon", "ruff_db", "ruff_diagnostics", + "ruff_python_ast", "ruff_ranged_value", "ruff_text_size", "salsa", @@ -3545,6 +3546,7 @@ dependencies = [ "ty_project", "ty_python_semantic", "ty_server", + "ty_site_packages", "ty_static", "walkdir", "wild", diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 68ba0e6ee7..d186c75d25 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -123,8 +123,17 @@ fn run_erased_union_phase<'a>( /// type-aware passes see only this file). Used for stdin input and tests; the /// file-backed [`transpile_typed`] resolves cross-module types. pub fn transpile(source: &str, config: &Config) -> Result { + transpile_with_report(source, config).map(|(output, _)| output) +} + +/// Like [`transpile`], and also reports what the emitted python needs installed +/// to run. +pub fn transpile_with_report( + source: &str, + config: &Config, +) -> Result<(String, RuntimeRequirements), String> { if config.is_python { - return Ok(source.to_owned()); + return Ok((source.to_owned(), RuntimeRequirements::default())); } // one db over the original source, shared by the qualification phase below @@ -197,7 +206,7 @@ pub fn transpile(source: &str, config: &Config) -> Result { } // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- - let final_output = run_import_redirect_phase(output, config); + let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &model.eagerly_imported_modules()); @@ -207,7 +216,7 @@ pub fn transpile(source: &str, config: &Config) -> Result { verify_syntax(&final_output).map_err(|e| e.message)?; verify_target_syntax(&final_output, config).map_err(|e| e.message)?; - Ok(final_output) + Ok((final_output, requirements)) } /// Transpile using ty's full type inference. `db` and `file` must already @@ -243,12 +252,71 @@ pub fn transpile_typed_with_map( config: &Config, rebuild: Option>, ) -> Result<(String, Vec>), TranspileError> { + transpile_typed_with_report(db, file, config, rebuild) + .map(|(output, line_map, _)| (output, line_map)) +} + +/// What the emitted python needs at run time that the standard library does not +/// provide. +/// +/// Lowering for an older python can put a name in the output that only +/// `typing_extensions` has there — `Self` on 3.9, say. That is a real dependency +/// of the built artifact, and nothing in the source says so, which is why the +/// transpile is what reports it: a wheel that shipped without it would install +/// cleanly and fail on the first import. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct RuntimeRequirements { + typing_extensions: bool, +} + +/// What the emitted python needs when lowering reached for `typing_extensions`. +/// +/// One floor for every name rather than one per name: which release of +/// `typing_extensions` first carried each backport is not something the emitted +/// code records, and a floor too low is a wheel that installs and then fails on +/// an import. It has to cover every name +/// [`ty_python_semantic::basedpython_typing_added_in`] can redirect, which is why +/// it lives beside the pass that does the redirecting rather than beside the +/// command that reports it. +const TYPING_EXTENSIONS_REQUIREMENT: &str = "typing_extensions>=4.12"; + +impl RuntimeRequirements { + /// Fold in what another module needed. + pub fn merge(&mut self, other: Self) { + self.typing_extensions |= other.typing_extensions; + } + + /// The requirements, spelled the way a `[project] dependencies` entry is. + /// + /// A list rather than a set of flags, so that a second requirement is a line + /// here instead of an edit at every place a caller asks what is needed. + pub fn specifiers(self) -> Vec<&'static str> { + let mut specifiers = Vec::new(); + if self.typing_extensions { + specifiers.push(TYPING_EXTENSIONS_REQUIREMENT); + } + specifiers + } +} + +/// Like [`transpile_typed_with_map`], and also reports what the emitted python +/// needs installed to run. +pub fn transpile_typed_with_report( + db: &dyn ty_python_semantic::Db, + file: File, + config: &Config, + rebuild: Option>, +) -> Result<(String, Vec>, RuntimeRequirements), TranspileError> { let source_ref = ruff_db::source::source_text(db, file); let original_source = source_ref.as_str(); if config.is_python { let out = original_source.to_owned(); - return Ok((out, source_map::line_table(original_source, &[]))); + return Ok(( + out, + source_map::line_table(original_source, &[]), + RuntimeRequirements::default(), + )); } // erased-union reification: give a `list[int] | list[str]` parameter a @@ -364,7 +432,7 @@ pub fn transpile_typed_with_map( return Err(first.clone().into()); } - let final_output = run_import_redirect_phase(output, config); + let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &eager_imports); let final_output = run_version_polyfill_phase(final_output, config); @@ -410,7 +478,7 @@ pub fn transpile_typed_with_map( return Err(err); } - Ok((final_output, line_map)) + Ok((final_output, line_map, requirements)) } fn newline_count(s: &str) -> usize { @@ -499,7 +567,7 @@ fn run_anon_named_tuple_cleanup(mut source: String, config: &Config) -> Result String { +fn run_import_redirect_phase(source: String, config: &Config) -> (String, RuntimeRequirements) { let (db, file) = make_in_memory_db(&source); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); @@ -515,11 +583,16 @@ fn run_import_redirect_phase(source: String, config: &Config) -> String { } if typing_redirect.edits.is_empty() { - return source; + return (source, RuntimeRequirements::default()); } let (output, _) = apply_transforms_once(src, typing_redirect.edits); - output + ( + output, + RuntimeRequirements { + typing_extensions: true, + }, + ) } /// Lazy-import marking phase: walks the post-typing-redirect output and diff --git a/crates/ty/Cargo.toml b/crates/ty/Cargo.toml index 10fdd4c652..ac11bfa21d 100644 --- a/crates/ty/Cargo.toml +++ b/crates/ty/Cargo.toml @@ -24,6 +24,7 @@ doctest = false [dependencies] ruff_db = { workspace = true, features = ["os", "cache", "junit"] } +ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } ruff_diagnostics = { workspace = true } by_build = { workspace = true } @@ -37,6 +38,7 @@ ty_ide = { workspace = true } ty_project = { workspace = true, features = ["zstd", "junit"] } ty_python_semantic = { workspace = true, features = ["serde"] } ty_server = { workspace = true } +ty_site_packages = { workspace = true } ty_static = { workspace = true } anyhow = { workspace = true } @@ -66,7 +68,6 @@ tikv-jemallocator = { workspace = true } [dev-dependencies] ruff_db = { workspace = true, features = ["testing"] } -ruff_python_ast = { workspace = true } ruff_python_trivia = { workspace = true } ty_module_resolver = { workspace = true } ty_python_core = { workspace = true } diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index ef6ae1a085..1fbd5838b7 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -19,7 +19,8 @@ by
by version

Display ty's version

by explain

Explain rules and other parts of ty

by run

Transpile and run a module with python -m <module>

-
by build

Transpile all .by files and write them to out/

+
by init

Start a new project

+
by build

Build the project as python

by compile

Compile .by and .py files to native CPython extension modules

by generate-api-file

Generate an api lockfile (api.lock) summarising the public type-level surface of the project

by transpile

Transpile a file to stdout, or a whole directory in place (reads stdin if no path given)

@@ -247,13 +248,43 @@ by run [OPTIONS] [MODULE] [ARGS]...
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--python, --venv path

The interpreter to run on, or the environment holding it.

+

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

[default: default]

+## by init + +Start a new project. + +Writes a `pyproject.toml` that names the basedpython build backend, a `src` layout, and a python version the checker, the transpiler and the interpreter all agree on — so the project is installable, runnable and publishable from the moment it exists. + +

Usage

+ +``` +by init [OPTIONS] [PATH] +``` + +

Arguments

+ +
PATH

Where to create the project [default: the current directory]

+
+ +

Options

+ +
--app

Create an application, with an entry point by run uses. The default

+
--help, -h

Print help (see a summary with '-h')

+
--lib

Create a library: no entry point, the same packaging

+
--name name

The project's name [default: the directory's name]

+
--python-version version

The python version to target [default: the version of the project environment's interpreter]

+
+ ## by build -Transpile all .by files and write them to out/ +Build the project as python. + +The output is the whole project, not only the transpiled half: every `.by` file becomes a `.py`, and every other file — a hand-written `.py`, a `py.typed`, a template, a data file — is carried over to the same place. What the previous build wrote and this one did not is deleted.

Usage

@@ -263,12 +294,17 @@ by build [OPTIONS]

Options

-
--help, -h

Print help

+
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the project's configured python version]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--out, -o dir

Where to write the output [default: out, or dist with --wheels]

+
--print-manifest

Report what the build read and produced, as <kind> <value> lines.

+

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

+

[default: default]

--wheels

Build one publishable wheel per python version, and a source distribution, into dist/.

+

Each wheel is lowered to the version it is tagged for, so an installer hands every interpreter the best wheel it can use rather than one lowered to the oldest python the project supports. Needs uv, which does the packaging.

+
## by compile diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index 85ff559aec..6884915c55 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -846,6 +846,190 @@ Defaults to `false`. --- +## `build` + +### `exclude` + +Files to keep out of the build output. + +The syntax is the same as `src.exclude`, and paths are anchored to the +project root. Excluding a `.by` file keeps its transpiled output out of +the build as well. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + exclude = [ + "tests", + "**/*.snapshot", + ] + ``` + +=== "ty.toml" + + ```toml + [build] + exclude = [ + "tests", + "**/*.snapshot", + ] + ``` + +--- + +### `include` + +Files to carry into the build output verbatim, in addition to the ones +that are there by default. + +`by build` mirrors the whole module tree: a `.by` file is transpiled, and +every other file — a hand-written `.py`, a `py.typed` marker, a template, +a data file — is copied to the same place in the output. `include` is for +the files that sit *outside* a module root and still belong in the build, +such as a data directory next to `src`. + +The syntax is the same as `src.include`, and paths are anchored to the +project root. `exclude` takes precedence over `include`. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + include = [ + "assets", + ] + ``` + +=== "ty.toml" + + ```toml + [build] + include = [ + "assets", + ] + ``` + +--- + +### `sources` + +Whether the build output carries the `.by` sources alongside the python +they were transpiled into, with a `by.typed` marker naming them as the +authoritative surface. + +This is what lets one basedpython project depend on another: a downstream +python project reads the transpiled `.py` and is served perfectly, while a +downstream basedpython project reads the `.by` and keeps the declarations +that have no python spelling — `extension` blocks, `raises` clauses, +read-only `let`, sum types. + +Enabled by default. Turn it off to ship python only. + +**Default value**: `true` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + sources = false + ``` + +=== "ty.toml" + + ```toml + [build] + sources = false + ``` + +--- + +### `version-from` + +The module to read `__version__` from, when `[project]` declares +`dynamic = ["version"]`. + +This is read when a wheel or a source distribution is built, not by the +checker: a version has to be settled before the packaging backend sees the +project, and the place it lives is a `.by` module that backend cannot +read. + +The value is a path relative to the project root. + +**Default value**: `null` + +**Type**: `str` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + version-from = "src/app/__init__.by" + ``` + +=== "ty.toml" + + ```toml + [build] + version-from = "src/app/__init__.by" + ``` + +--- + +### `wheel-versions` + +The python versions to build a wheel for, one wheel each. + +`by build --wheels` builds one wheel per version listed and tags each for +the python it was lowered to, so an installer hands every interpreter the +best wheel it can use. A python with no wheel of its own takes the newest +one below it. + +Defaults to every version from the one the project targets up to the +newest this release knows about — which is what `requires-python` already +says the project supports, so most projects need not set this. List them +explicitly to ship fewer. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + wheel-versions = ["3.9", "3.13"] + ``` + +=== "ty.toml" + + ```toml + [build] + wheel-versions = ["3.9", "3.13"] + ``` + +--- + ## `editor` ### `common-aliases` diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 9a0b1f9d13..54bead8781 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -59,6 +59,13 @@ Used to determine the name of the active Conda environment. Used to detect the path of an active Conda environment. If both `VIRTUAL_ENV` and `CONDA_PREFIX` are present, `VIRTUAL_ENV` will be preferred. +### `PYTHON` + +The interpreter `by run` executes the transpiled program on. + +Overrides the project environment, and is itself overridden by +`by run --python`. + ### `PYTHONPATH` Adds additional directories to ty's search paths. diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 3c9cfae9d3..0063da1721 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -81,6 +81,12 @@ pub(crate) enum Command { /// [default: the version of the interpreter that will run it] #[arg(long, value_name = "VERSION")] min_version: Option, + /// The interpreter to run on, or the environment holding it. + /// + /// Defaults to the project environment — the same one `by check` + /// resolves imports against — then `$PYTHON`, then `python3` on `PATH`. + #[arg(long, value_name = "PATH", alias = "venv")] + python: Option, #[command(flatten)] lowering: LoweringArgs, /// Compile every imported module to a native extension first. @@ -95,12 +101,61 @@ pub(crate) enum Command { compiled: bool, }, - /// Transpile all .by files and write them to out/. + /// Start a new project. + /// + /// Writes a `pyproject.toml` that names the basedpython build backend, a + /// `src` layout, and a python version the checker, the transpiler and the + /// interpreter all agree on — so the project is installable, runnable and + /// publishable from the moment it exists. + Init { + /// Where to create the project [default: the current directory] + #[arg(value_name = "PATH")] + path: Option, + /// The project's name [default: the directory's name] + #[arg(long, value_name = "NAME")] + name: Option, + /// Create a library: no entry point, the same packaging. + #[arg(long, conflicts_with = "app")] + lib: bool, + /// Create an application, with an entry point `by run` uses. The default. + #[arg(long)] + app: bool, + /// The python version to target + /// [default: the version of the project environment's interpreter] + #[arg(long, value_name = "VERSION")] + python_version: Option, + }, + + /// Build the project as python. + /// + /// The output is the whole project, not only the transpiled half: every `.by` + /// file becomes a `.py`, and every other file — a hand-written `.py`, a + /// `py.typed`, a template, a data file — is carried over to the same place. + /// What the previous build wrote and this one did not is deleted. Build { /// minimum Python version the output must run on /// [default: the project's configured python version] #[arg(long, value_name = "VERSION")] min_version: Option, + /// Build one publishable wheel per python version, and a source + /// distribution, into `dist/`. + /// + /// Each wheel is lowered to the version it is tagged for, so an + /// installer hands every interpreter the best wheel it can use rather + /// than one lowered to the oldest python the project supports. Needs + /// `uv`, which does the packaging. + #[arg(long, conflicts_with_all = ["min_version", "print_manifest"])] + wheels: bool, + /// Where to write the output [default: `out`, or `dist` with `--wheels`] + #[arg(short = 'o', long, value_name = "DIR")] + out: Option, + /// Report what the build read and produced, as ` ` lines. + /// + /// `input ` for every file the project is made of — what a source + /// distribution has to carry to rebuild into the same thing — and + /// `package ` for every top-level package that came out. + #[arg(long)] + print_manifest: bool, #[command(flatten)] lowering: LoweringArgs, }, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 2f15a62359..439343b68d 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::ffi::OsStr; use std::fs; use std::io::{self, Read}; @@ -17,39 +17,28 @@ use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; use ruff_text_size::TextRange; use sha2::{Digest, Sha256}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; +use ty_site_packages::{PythonEnvironment, SysPrefixPathOrigin}; +use ty_static::EnvVars; use walkdir::WalkDir; use crate::ExitStatus; use crate::args::LoweringArgs; - -/// The python version the emitted code must run on when `--min-version` is not -/// given: the one the project configures (`environment.python-version`, else the -/// `requires-python` lower bound), so the checker and the emitter agree about -/// which python this project targets. Falls back to the transpiler's own default -/// outside a project, or when the configuration names no version. -fn configured_min_version(cwd: &Path) -> PythonVersion { - let Some(sys_cwd) = SystemPath::from_std_path(cwd) else { - return Config::default().min_version; - }; - let system = OsSystem::new(sys_cwd); - let Ok(metadata) = ProjectMetadata::discover(sys_cwd, &system) else { - return Config::default().min_version; - }; - let db = ProjectDatabase::use_defaults(metadata, system); - db.project() - .program(&db) - .python_version(&db) - .to_string() - .parse() - .unwrap_or_else(|_| Config::default().min_version) -} +use crate::by_staging::{Staging, relative_destination, transpiled_destination}; /// The transpile config for a command whose `--min-version` is optional. +/// +/// Without the flag the target is the version the project configures, so that the +/// checker and the emitter agree about which python this project is for. Outside a +/// project — `by transpile` reading a lone file — the transpiler's own default +/// stands in. fn version_config(min_version: Option<&str>, cwd: &Path) -> anyhow::Result { match min_version { Some(spelled) => parse_version(spelled), None => Ok(Config { - min_version: configured_min_version(cwd), + min_version: ResolvedProject::discover(cwd).map_or_else( + |_| Config::default().min_version, + |project| project.python_version(), + ), ..Config::default() }), } @@ -122,8 +111,15 @@ pub(crate) fn cmd_run( min_version: Option<&str>, lowering: &LoweringArgs, compiled: bool, + python_flag: Option<&Path>, ) -> anyhow::Result { - let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let cwd = std::env::current_dir().context("failed to get current directory")?; + // one resolution of the project, for the environment and the target version + // both: they are two readings of the same configuration and must not be able + // to disagree + let project = ResolvedProject::discover(&cwd)?; + let interpreter = discover_interpreter(python_flag, &project)?; + let python = interpreter.path.clone(); // `run` executes on a specific interpreter, so by default target *its* // version: the emitted code (dataclass `slots=`, PEP 695 syntax, …) must // match what that python actually supports. an explicit `--min-version` @@ -133,13 +129,13 @@ pub(crate) fn cmd_run( let mut config = match (min_version, probed) { (Some(flag), probed) => { let config = parse_version(flag)?; - if let Some(interpreter) = probed - && config.min_version > interpreter + if let Some(found) = probed + && config.min_version > found { anyhow::bail!( - "--min-version {flag} is newer than `{python}` ({interpreter}), \ - which could not run the emitted code — \ - set PYTHON to a {flag}+ interpreter" + "--min-version {flag} is newer than the interpreter this would run on: \ + `{python}` is {found}, from {}", + interpreter.origin ); } config @@ -150,11 +146,26 @@ pub(crate) fn cmd_run( }, (None, None) => Config::default(), }; + // a program written for the python the project declares cannot be run by an + // older one: the source itself may use syntax that python has no lowering + // for (`match`, for one), and the failure lands as a `SyntaxError` inside + // generated code rather than as anything the author can act on + if let Some(found) = probed { + let configured = project.python_version(); + if min_version.is_none() && found < configured { + anyhow::bail!( + "this project targets python {configured}, but the interpreter this would run on \ + is {found}: `{python}`, from {}\n \ + use an interpreter that is {configured} or newer, or pass \ + `--min-version {found}` to build for this one", + interpreter.origin + ); + } + } lowering.apply(&mut config)?; - let cwd = std::env::current_dir().context("failed to get current directory")?; let tmp = tempfile::TempDir::new().context("failed to create temp directory")?; - let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES)?; + let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, None)?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Failure); @@ -178,23 +189,30 @@ pub(crate) fn cmd_run( // each generated `.py` paired with its source `.by` and the line table that // lifts generated line numbers back to `.by` lines (for traceback rewriting) let mut traceback_entries: Vec = Vec::new(); + let mut staging = Staging::new(tmp.path()); let ok = render_check_and_transpile( &db, &handles, &config, CheckGate::AllErrors, &rebuilder, + &mut by_transforms::RuntimeRequirements::default(), |emitted| { - let py = tmp - .path() - .join(module_relative_path(&roots, &root, emitted.by_path)); - traceback_entries.push(write_module(py, emitted)?); + let relative = transpiled_destination(&roots, &root, emitted.by_path); + traceback_entries.push(stage_module(&mut staging, &relative, emitted)?); Ok(()) }, )?; if !ok { return Ok(ExitStatus::Failure); } + // a program is its data as much as its modules: a `.py` module it imports, a + // json file it opens, a template it renders. running out of a directory + // holding only the transpiled half fails on the first of them + stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_by_typed_markers(&db, &mut staging, &roots, &root)?; + write_traceback_runtime(&mut staging, &traceback_entries)?; + staging.finish()?; if compiled { // the extension lands beside the generated `.py`, and python's finder @@ -258,15 +276,18 @@ pub(crate) fn cmd_run( } } - write_traceback_runtime(tmp.path(), &traceback_entries)?; - let status = Command::new(&python) .arg(BY_RUNNER_FILENAME) .arg(&module) .args(args) .current_dir(tmp.path()) .status() - .with_context(|| format!("{python}: failed to execute"))?; + .with_context(|| { + format!( + "could not run `{python}`, the interpreter from {}", + interpreter.origin + ) + })?; let code = status.code().unwrap_or(1); // drop the temp dir explicitly: `process::exit` skips destructors, so @@ -293,37 +314,162 @@ fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { roots } -/// Where `bpy`'s transpiled python goes, relative to the output root. +/// The source roots a distribution's packages come from. /// -/// The tree mirrored is the *module* tree, not the directory tree: a src-layout -/// project's `src/pkg/main.by` is the module `pkg.main`, so it has to land at -/// `pkg/main.py`. Mirroring the directory instead emits `src/pkg/main.py`, -/// whose module is `src.pkg.main` — a name nothing imports, and one `run.main` -/// cannot sensibly be set to. -fn module_relative_path(roots: &[PathBuf], root: &Path, bpy: &Path) -> PathBuf { - let relative = roots +/// The project root is always a module root — it is what lets `tests/` and a +/// script beside it resolve their imports — but for a src-layout project it is +/// not where the *distribution* lives. `src/app` is a package of this project; +/// `tests` beside it is not something anyone installs. So when the project +/// declares somewhere for its modules to live, that is where they live, and the +/// root counts only when it is the only answer. +fn packaging_roots(roots: &[PathBuf], root: &Path) -> Vec { + let declared: Vec = roots .iter() - .find_map(|candidate| bpy.strip_prefix(candidate).ok()) - .or_else(|| bpy.strip_prefix(root).ok()) - .unwrap_or(bpy); - // whatever happened above, the result has to be *relative*: joined onto the - // output directory an absolute path replaces it outright, so every emitted - // file would land outside the output tree. keeping only the named components - // also drops any `..`, which would climb back out of it - relative - .components() - .filter_map(|component| match component { - std::path::Component::Normal(name) => Some(name), - _ => None, + .filter(|candidate| candidate.as_path() != root) + .cloned() + .collect(); + if declared.is_empty() { + vec![root.to_path_buf()] + } else { + declared + } +} + +/// The top-level packages the build produced, as a distribution would ship them. +fn staged_packages(staging: &Staging, roots: &[PathBuf], root: &Path) -> Vec { + let packaging = packaging_roots(roots, root); + let mut packages: Vec = staging + .entries() + .filter(|(_, source)| { + source.is_some_and(|source| { + packaging + .iter() + .any(|candidate| source.starts_with(candidate)) + }) }) - .collect::() - .with_extension("py") + .filter_map(|(destination, _)| { + let mut components = destination.components(); + let package = components.next()?; + let rest: PathBuf = components.collect(); + matches!( + rest.to_str(), + Some("__init__.py" | "__init__.pyi" | "__init__.by" | "__init__.byi") + ) + .then(|| package.as_os_str().to_str().map(str::to_owned)) + .flatten() + }) + .collect(); + packages.sort(); + packages.dedup(); + packages +} + +/// Carry every file the transpiler did not produce into the output tree. +/// +/// This is what makes the output a project rather than a heap of transpiled +/// modules: a hand-written `.py` sibling, a `py.typed`, a template, a fixture — +/// all of it lands in the same relative place, so the output imports and reads +/// data exactly the way the source tree does. +fn stage_verbatim( + db: &ProjectDatabase, + root: &Path, + roots: &[PathBuf], + staging: &mut Staging, +) -> anyhow::Result<()> { + let settings = db.project().settings(db); + let build = settings.build(); + // a file the project excludes from itself is not part of what it ships, so + // `src.exclude` bounds the build and `build.exclude` narrows it further + let src = settings.src(); + let out = staging.out().to_path_buf(); + + for entry in WalkDir::new(root) + .into_iter() + .filter_entry(|entry| { + // the output tree is not an input to itself, wherever `--out` put it + if entry.path() == out { + return false; + } + if !may_hold_build_content(entry) { + return false; + } + !entry.file_type().is_dir() + || SystemPath::from_std_path(entry.path()).is_none_or(|path| { + build.is_directory_included(path) && src.is_directory_included(path) + }) + }) + .filter_map(Result::ok) + { + let path = entry.path(); + if entry.path_is_symlink() || !entry.file_type().is_file() { + continue; + } + let extension = path.extension().and_then(OsStr::to_str); + // a `.by` is an input, and it is carried over only to be read by a + // downstream basedpython project — never for python to import + if matches!(extension, Some("by" | "byi")) && !build.sources() { + continue; + } + let Some(system_path) = SystemPath::from_std_path(path) else { + continue; + }; + if !build.is_file_included(system_path) || !src.is_file_included(system_path) { + continue; + } + staging.copy(&relative_destination(roots, root, path), path)?; + } + Ok(()) +} + +/// Write the `by.typed` marker into every package the build ships. +/// +/// The marker says two things to a project that installs this one, and both are +/// things nothing else can tell it. Its presence says the `.by` beside a module is +/// the authoritative surface, to be read in preference to the python it was +/// transpiled into — the same bargain `py.typed` strikes for inline annotations. +/// Its contents say which of this project's dependencies are part of its own +/// interface, which a `pyproject.toml` cannot, because nothing installs one. +/// +/// Only the packages the project *ships* are marked, and they are read off what +/// was written rather than off the source layout — a `tests` package beside `src` +/// is neither shipped nor anybody else's business. +#[allow(clippy::print_stderr)] +fn stage_by_typed_markers( + db: &ProjectDatabase, + staging: &mut Staging, + roots: &[PathBuf], + root: &Path, +) -> anyhow::Result<()> { + let exported = db + .project() + .settings(db) + .analysis() + .exported_dependencies + .clone() + .unwrap_or_default(); + // written whether or not the `.by` sources went with it: the precedence claim + // is vacuous without them — nothing to prefer — but the export declaration is + // not, and a python-only build still has dependencies it hands out on purpose + let marker = ty_module_resolver::Marker::render(&exported); + + for package in staged_packages(staging, roots, root) { + staging.write( + &Path::new(&package).join(ty_module_resolver::BY_TYPED), + None, + &marker, + )?; + } + + if !exported.is_empty() { + eprintln!("exporting {}", exported.join(", ")); + } + Ok(()) } /// The dotted module name a file laid out at `relative` will be imported under. /// /// The tree the generated python is written into *is* the module tree — every -/// file lands at [`module_relative_path`] — so the name is that path with its +/// file lands at [`transpiled_destination`] — so the name is that path with its /// separators turned into dots. `pkg/__init__.py` is the package `pkg` itself, /// which is the name a class defined in it reports as its `__module__`. /// @@ -410,6 +556,222 @@ fn configured_main(db: &ProjectDatabase) -> Option { Some((**main).clone()) } +/// The python version a brand new project should target. +/// +/// The environment it will be developed in is the right answer when there is +/// one: the version the checker targets, the version the transpiler emits for, +/// and the version that runs the result should be one version rather than three. +/// A project being created usually has no environment yet, though, and the bare +/// `python3` that turns up on `PATH` instead is whatever the operating system +/// shipped years ago. Pinning a new project to that is how a project ends up +/// targeting 3.9 for its whole life without anyone choosing to. +pub(crate) fn default_project_python_version() -> PythonVersion { + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + // `by init` runs before there is a project to resolve, so a failure here is + // the ordinary case rather than a problem + let Ok(project) = ResolvedProject::discover(&cwd) else { + return latest_python_version(); + }; + let Ok(interpreter) = discover_interpreter(None, &project) else { + return latest_python_version(); + }; + if interpreter.is_from_path { + return latest_python_version(); + } + detect_python_version(&interpreter.path).unwrap_or_else(|| Config::default().min_version) +} + +/// The newest python this release can emit for. +fn latest_python_version() -> PythonVersion { + ruff_python_ast::PythonVersion::latest() + .to_string() + .parse() + .unwrap_or_else(|_| Config::default().min_version) +} + +/// The interpreter `by run` executes on, and how it was chosen. +#[derive(Clone)] +struct Interpreter { + path: String, + origin: String, + /// whether this is the bare `python3` off `PATH` — the last resort, and the + /// only origin that says nothing about what the project targets + is_from_path: bool, +} + +/// Find the interpreter to run the program on. +/// +/// The project environment comes first, because that is the environment the +/// project *is*: `by check` resolved this project's imports against it, so +/// running against a different python answers a question nobody asked. That has +/// to mean the same environment the checker used, resolved the same way — the +/// `environment.python` the project configures, then an activated virtual +/// environment, a conda environment, or a `.venv` beside the project's +/// `pyproject.toml`. +/// +/// The project root is what all of that is relative to, not the working +/// directory: `by run` from a subdirectory is still this project, and its `.venv` +/// is still the one at the top. +/// +/// `--python` overrides everything, for one run. `$PYTHON` is below discovery +/// because it names an interpreter rather than an environment, so it stands in +/// only where there is no project environment to prefer — but it still beats the +/// bare `python3` that discovery falls back to. +fn discover_interpreter( + flag: Option<&Path>, + project: &ResolvedProject, +) -> anyhow::Result { + let named = |path: String, origin: &str| Interpreter { + path, + origin: origin.to_owned(), + is_from_path: false, + }; + + if let Some(flag) = flag { + // a `--python` may name the interpreter itself or the environment it + // lives in, the same way `by check --python` does + if flag.is_file() { + return Ok(named(flag.display().to_string(), "`--python`")); + } + if let Some(interpreter) = + interpreter_in_environment(flag, SysPrefixPathOrigin::PythonCliFlag) + { + return Ok(named(interpreter, "`--python`")); + } + return Ok(named(flag.display().to_string(), "`--python`")); + } + + // what the project says its environment is, which is what the checker used. + // a configured environment that cannot be resolved is an error rather than + // something to fall past: `by check` refuses it outright, and running on a + // different python than the one just type-checked against — reporting it as + // a version mismatch, which names the wrong cause — is how the two commands + // came to disagree in the first place + if let Some(configured) = project.configured_environment() { + let Some(interpreter) = + interpreter_in_environment(&configured, SysPrefixPathOrigin::PythonCliFlag) + else { + anyhow::bail!( + "`environment.python` is `{}`, which is not a python environment — \ + the same setting `by check` reads, so neither command can use it", + configured.display() + ); + }; + return Ok(named(interpreter, "`environment.python`")); + } + + let discovered = discovered_environment(project.root()); + if let Some(found) = &discovered + && !found.is_from_path + { + return Ok(found.clone()); + } + + if let Ok(python) = std::env::var(EnvVars::PYTHON) { + return Ok(named(python, "`PYTHON`")); + } + + Ok(discovered.unwrap_or_else(|| Interpreter { + path: "python3".to_owned(), + origin: "`PATH`".to_owned(), + is_from_path: true, + })) +} + +/// A project, resolved once. +/// +/// Discovery walks up from the working directory reading configuration, and +/// every answer taken from it — where the root is, which environment the project +/// declares, which python it targets — has to be the same answer. Resolving it +/// per question was not only repeated work: the copies disagreed about failure, +/// one falling back to the working directory where another gave up. +pub(crate) struct ResolvedProject { + root: PathBuf, + metadata: ProjectMetadata, +} + +impl ResolvedProject { + fn discover(cwd: &Path) -> anyhow::Result { + let sys_cwd = SystemPath::from_std_path(cwd) + .with_context(|| format!("non-utf8 path: {}", cwd.display()))?; + let system = OsSystem::new(sys_cwd); + let metadata = ProjectMetadata::discover(sys_cwd, &system) + .with_context(|| format!("failed to discover project at {sys_cwd}"))?; + let root = PathBuf::from(metadata.root().as_str()); + Ok(Self { root, metadata }) + } + + fn root(&self) -> &Path { + &self.root + } + + /// The environment the project configures, as an absolute path. + fn configured_environment(&self) -> Option { + let sys_root = SystemPath::from_std_path(&self.root)?; + let system = OsSystem::new(sys_root); + let configured = self + .metadata + .options() + .environment + .as_ref()? + .python + .as_ref()?; + Some(PathBuf::from( + configured.absolute(sys_root, &system).as_str(), + )) + } + + /// The python version the emitted code must run on: the one the project + /// configures (`environment.python-version`, else the `requires-python` lower + /// bound), so the checker and the emitter agree about which python this + /// project targets. + fn python_version(&self) -> PythonVersion { + let Some(sys_root) = SystemPath::from_std_path(&self.root) else { + return Config::default().min_version; + }; + let system = OsSystem::new(sys_root); + let db = ProjectDatabase::use_defaults(self.metadata.clone(), system); + db.project() + .program(&db) + .python_version(&db) + .to_string() + .parse() + .unwrap_or_else(|_| Config::default().min_version) + } +} + +/// The environment discovery finds for this project: an activated virtual +/// environment, a conda environment, or a `.venv` at the project root — and, +/// failing all of those, whatever python is on `PATH`. +fn discovered_environment(root: &Path) -> Option { + let sys_root = SystemPath::from_std_path(root)?; + let system = OsSystem::new(sys_root); + let environment = PythonEnvironment::discover(sys_root, &system).ok()??; + let interpreter = environment.interpreter(&system)?; + // discovery ends by falling back to whatever python is on `PATH`, which is + // an interpreter but not a *project* environment — the difference is what + // decides whether it outranks `$PYTHON`, and what a new project targets + let is_from_path = matches!( + environment.origin(), + SysPrefixPathOrigin::PythonBinary | SysPrefixPathOrigin::SelfEnvironment + ); + Some(Interpreter { + path: interpreter.to_string(), + origin: environment.origin().to_string(), + is_from_path, + }) +} + +/// The interpreter inside the environment rooted at `path`, if there is one. +fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Option { + let sys_path = SystemPath::from_std_path(path)?; + let system = OsSystem::new(sys_path); + let environment = PythonEnvironment::new(sys_path, origin, &system).ok()?; + environment + .interpreter(&system) + .map(|interpreter| interpreter.to_string()) +} + /// Probe `python`'s `major.minor` version (e.g. `3.9`) so `run` can target the /// interpreter it will execute on. Returns `None` if the interpreter can't be /// run or its output can't be parsed. @@ -431,40 +793,46 @@ fn detect_python_version(python: &str) -> Option { pub(crate) fn cmd_build( min_version: Option<&str>, lowering: &LoweringArgs, + out: &Path, + print_manifest: bool, ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let mut config = version_config(min_version, &cwd)?; lowering.apply(&mut config)?; - let out = cwd.join("out"); - let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES)?; + // the output directory is settled before the project is read, because it is + // the one directory the project must not be read *from*: it holds a copy of + // every source this build is about to write. canonical, because that is what + // the paths it is compared against are — creating it first is what makes + // canonicalizing it possible + let out = cwd.join(out); + fs::create_dir_all(&out).with_context(|| format!("could not create {}", out.display()))?; + let out = fs::canonicalize(&out).unwrap_or(out); + + let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, Some(&out))?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Success); } let file_count = handles.len(); let roots = module_roots(&db, &root); - let mut packages: BTreeSet = BTreeSet::new(); + let mut staging = Staging::new(&out); // `out/` outlives the build that wrote it — it is what a test runner, a // debugger or an editor plugin sees — so the sourcemap goes with it. this is // the directory where a `.by` really can be saved after the transpile, which // is what the digests beside the map are for let mut entries: Vec = Vec::new(); + let mut requirements = by_transforms::RuntimeRequirements::default(); if !render_check_and_transpile( &db, &handles, &config, CheckGate::ParseErrorsOnly, &rebuilder, + &mut requirements, |emitted| { - let relative = module_relative_path(&roots, &root, emitted.by_path); - if relative.components().count() > 1 - && let Some(package) = relative.components().next() - { - packages.insert(out.join(package)); - } - - let entry = write_module(out.join(relative), emitted)?; + let relative = transpiled_destination(&roots, &root, emitted.by_path); + let entry = stage_module(&mut staging, &relative, emitted)?; eprintln!( "{} -> {}", emitted.by_path.display(), @@ -476,41 +844,49 @@ pub(crate) fn cmd_build( )? { return Ok(ExitStatus::Failure); } - write_sourcemap_module(&out, &entries)?; - write_markers(&db, &packages)?; + stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_by_typed_markers(&db, &mut staging, &roots, &root)?; + write_sourcemap_module(&mut staging, &entries)?; + if print_manifest { + print_build_manifest(&staging, &roots, &root, requirements)?; + } + staging.finish()?; eprintln!("\nbuild complete ({file_count} files)"); Ok(ExitStatus::Success) } -/// Writes the `by.typed` marker into every package the build emitted. +/// Report what the build read and what it produced, as ` ` lines. /// -/// The marker is what tells a project that installs this one that its packages -/// are basedpython's, and it carries the one thing a `pyproject.toml` cannot tell -/// them: which of this project's dependencies are part of its own interface. -/// Nothing installs a `pyproject.toml`, and this rides along inside the package. -#[allow(clippy::print_stderr)] -fn write_markers(db: &ProjectDatabase, packages: &BTreeSet) -> anyhow::Result<()> { - let exported = db - .project() - .settings(db) - .analysis() - .exported_dependencies - .clone() - .unwrap_or_default(); - let marker = ty_module_resolver::Marker::render(&exported); +/// Two questions, both of which only the build can answer: which files this +/// project is made of — a source distribution has to carry exactly those, since +/// they are what rebuilds into the same wheel — and which top-level packages came +/// out. Answering them here rather than in the packaging layer keeps one answer +/// to "what is this project", instead of a second one that has to be kept in step. +#[allow(clippy::print_stdout)] +fn print_build_manifest( + staging: &Staging, + roots: &[PathBuf], + root: &Path, + requirements: by_transforms::RuntimeRequirements, +) -> anyhow::Result<()> { + use std::io::Write as _; - for package in packages { - let path = package.join(ty_module_resolver::BY_TYPED); - fs::create_dir_all(package)?; - fs::write(&path, &marker)?; + let mut stdout = io::stdout().lock(); + for input in staging.inputs() { + let relative = input.strip_prefix(root).unwrap_or(input); + writeln!(stdout, "input {}", relative.display())?; } - - if !exported.is_empty() { - eprintln!("exporting {}", exported.join(", ")); + for package in staged_packages(staging, roots, root) { + writeln!(stdout, "package {package}")?; + } + // lowering for an older python can put a name in the output that only + // `typing_extensions` has there. nothing in the source says so, so nothing + // but the build can + for specifier in requirements.specifiers() { + writeln!(stdout, "requires {specifier}")?; } - Ok(()) } @@ -577,7 +953,7 @@ pub(crate) fn cmd_compile( // gradual, and `--no-any` would then fail on noise // `compile` embeds fallback source produced by the untyped transpile, which // takes no db, so the rebuilder the other commands thread through is unused here - let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES, None)?; // the database holds the whole project so a type imported from a sibling // resolves, but only the files that were *asked for* are checked and emitted. @@ -874,6 +1250,11 @@ const NON_SOURCE_DIRS: &[&str] = &[ "dist", "node_modules", "out", + // rust's build directory, which a basedpython project has whenever it also + // has an extension crate — and which the build would otherwise copy in full. + // a project that really does have a package called `target` can take it back + // with `exclude = ["!target"]` + "target", ]; fn cmd_transpile_dir(dir: &Path, reverse: bool, config: &Config) -> anyhow::Result { @@ -985,7 +1366,7 @@ fn reverse_dir_converting( /// build`, but written in place rather than to `out/`). #[allow(clippy::print_stderr)] fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { - let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES)?; + let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES, None)?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Success); @@ -997,6 +1378,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { config, CheckGate::ParseErrorsOnly, &rebuilder, + &mut by_transforms::RuntimeRequirements::default(), |emitted| { let py = emitted.by_path.with_extension("py"); fs::write(&py, emitted.python).with_context(|| format!("{}", py.display()))?; @@ -1028,6 +1410,24 @@ fn py_source_files(root: &Path) -> Vec { .collect() } +/// Whether the build walk may descend into this entry. +/// +/// Narrower than [`may_contain_sources`] on purpose, for the same reason +/// [`is_hidden_within`] is: this walk applies the project's own `src` and `build` +/// filters as it goes, so everything ty's `src.exclude` defaults already drop is +/// covered — and re-dropping it here would take back a file that a negated +/// exclude deliberately re-included. Only the directories ty's defaults *leave* +/// (and hidden ones) still have to be turned away. +fn may_hold_build_content(entry: &walkdir::DirEntry) -> bool { + if entry.depth() == 0 || !entry.file_type().is_dir() { + return true; + } + entry + .file_name() + .to_str() + .is_some_and(|name| !name.starts_with('.') && !NON_SOURCE_DIRS_TY_ALLOWS.contains(&name)) +} + /// Whether a project walk may descend into this entry: hidden directories /// (`.claude`, `.git`, `.venv`, …) and [`NON_SOURCE_DIRS`] never hold /// first-party source. The walk root itself is always entered, even when the @@ -1065,17 +1465,20 @@ struct TracebackEntry { py_digest: String, } -/// write one emitted module out and describe it, in that order: the digests -/// are over the bytes that just landed on disk -fn write_module(py_path: PathBuf, emitted: &Transpiled<'_>) -> anyhow::Result { - if let Some(parent) = py_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&py_path, emitted.python) - .with_context(|| format!("failed to write {}", py_path.display()))?; +/// stage one emitted module and describe it, in that order: the digests are over +/// the bytes that just landed on disk +/// +/// it goes through the staging rather than straight to the path, so that a module +/// two sources both claim is reported, and so that the file is one the manifest +/// knows about and a later build can clean up +fn stage_module( + staging: &mut Staging, + relative: &Path, + emitted: &Transpiled<'_>, +) -> anyhow::Result { + staging.write(relative, Some(emitted.by_path), emitted.python)?; Ok(TracebackEntry { - py_path, + py_path: staging.out().join(relative), by_path: fs::canonicalize(emitted.by_path) .unwrap_or_else(|_| emitted.by_path.to_path_buf()), line_map: emitted.line_map.to_vec(), @@ -1100,7 +1503,7 @@ fn content_digest(bytes: &[u8]) -> String { /// both tables are keyed by the generated path exactly as written here. a /// consumer that normalises those keys — the runner shim resolves symlinks, for /// one — has to keep the original key to reach the entry's digests -fn write_sourcemap_module(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Result<()> { +fn write_sourcemap_module(staging: &mut Staging, entries: &[TracebackEntry]) -> anyhow::Result<()> { use std::fmt::Write as _; let mut map_src = String::from( @@ -1146,20 +1549,21 @@ fn write_sourcemap_module(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Res } map_src.push_str("}\n"); - fs::create_dir_all(dir).with_context(|| format!("failed to create {}", dir.display()))?; - fs::write(dir.join(BY_SOURCEMAP_FILENAME), map_src) - .with_context(|| "failed to write sourcemap module")?; - Ok(()) + staging.write(Path::new(BY_SOURCEMAP_FILENAME), None, &map_src) } /// write the sourcemap module + runner shim into the run dir. the shim runs the /// target module and, on an uncaught exception, rewrites traceback frames in /// generated files back to their `.by` source location -fn write_traceback_runtime(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Result<()> { - write_sourcemap_module(dir, entries)?; - fs::write(dir.join(BY_RUNNER_FILENAME), BY_RUNNER_SRC) - .with_context(|| "failed to write runner shim")?; - Ok(()) +fn write_traceback_runtime( + staging: &mut Staging, + entries: &[TracebackEntry], +) -> anyhow::Result<()> { + write_sourcemap_module(staging, entries)?; + // through the staging, so that a project of its own with this name is + // reported as the collision it is rather than silently overwritten by a shim + // it knows nothing about + staging.write(Path::new(BY_RUNNER_FILENAME), None, BY_RUNNER_SRC) } /// Render a string as a python string literal (double-quoted, minimal escaping). @@ -1382,6 +1786,7 @@ const NON_SOURCE_DIRS_TY_ALLOWS: &[&str] = &[ ".pytest_cache", "build", "out", + "target", ]; /// Whether `path` sits inside a hidden or build-output directory under `root`. @@ -1402,7 +1807,11 @@ fn is_hidden_within(path: &Path, root: &Path) -> bool { /// extension is in `extensions` /// — the same set `by check` walks, so `src.exclude` and the ignore files it /// honours apply here too — and the means to build the same project again. -fn build_project_db(cwd: &Path, extensions: &[&str]) -> anyhow::Result { +fn build_project_db( + cwd: &Path, + extensions: &[&str], + output: Option<&Path>, +) -> anyhow::Result { // the project root must be canonicalized the same way the included files // are (below) so it stays a path *prefix* of them: otherwise a file's // search path isn't recognized as first-party and boundary diagnostics @@ -1415,6 +1824,17 @@ fn build_project_db(cwd: &Path, extensions: &[&str]) -> anyhow::Result anyhow::Result