From 2fbb1f8d5a581ca99db1a26c00c9fe600126eaa1 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:25:02 +1000 Subject: [PATCH 1/4] build the whole project, and ship it as a wheel `by build` and `by run` wrote only the transpiled `.by` files, so a project's hand-written `.py` modules and its data never reached the output. under `run` that was fatal: it executes out of the tree it stages, so a `.py` sibling could not be imported at all. both now stage the whole project, deleting what the previous build wrote and this one did not. `basedpython.build` is a PEP 517 backend, so `uv build` produces a wheel and an sdist end to end. it stages with `by build` and hands the result to `uv_build`. the wheel carries the transpiled python, the `.by` sources, and a `by.typed` marker; a package with that marker resolves to its `.by` rather than its `.py`, so a basedpython consumer keeps the declarations python cannot spell. `by run` also takes the project environment rather than whatever `python3` is on `PATH`, and `by init` writes a project already shaped to build. a source directory that is itself a package ships under its own name --- .github/workflows/ci.yaml | 3 + Cargo.lock | 1 + crates/basedpython/Cargo.lock | 2 + crates/ty/Cargo.toml | 3 +- crates/ty/docs/cli.md | 40 +- crates/ty/docs/configuration.md | 148 ++++ crates/ty/docs/environment.md | 7 + crates/ty/src/args.rs | 48 +- crates/ty/src/by_commands.rs | 531 +++++++++---- crates/ty/src/by_init.rs | 207 +++++ crates/ty/src/by_staging.rs | 402 ++++++++++ crates/ty/src/lib.rs | 24 +- crates/ty/tests/by_e2e.rs | 707 ++++++++++++++++++ crates/ty/tests/cli/config_option.rs | 2 +- crates/ty_module_resolver/src/list.rs | 7 +- crates/ty_module_resolver/src/path.rs | 45 ++ crates/ty_module_resolver/src/resolve.rs | 57 +- crates/ty_project/src/metadata/options.rs | 154 +++- crates/ty_project/src/metadata/settings.rs | 75 ++ .../mdtest/import/by_typed_packages.md | 229 ++++++ .../e2e__commands__debug_command.snap | 17 + crates/ty_site_packages/src/lib.rs | 35 + crates/ty_static/src/env_vars.rs | 6 + docs/basedpython/getting-started.md | 14 +- docs/basedpython/index.md | 2 +- docs/basedpython/packaging.md | 189 +++++ pyproject.toml | 12 +- python/basedpython/__init__.py | 12 + python/basedpython/build.py | 527 +++++++++++++ scripts/test_build_backend.py | 274 +++++++ ty.schema.json | 57 ++ zensical.toml | 3 + 32 files changed, 3673 insertions(+), 167 deletions(-) create mode 100644 crates/ty/src/by_init.rs create mode 100644 crates/ty/src/by_staging.rs create mode 100644 crates/ty_python_semantic/resources/mdtest/import/by_typed_packages.md create mode 100644 docs/basedpython/packaging.md create mode 100644 python/basedpython/__init__.py create mode 100644 python/basedpython/build.py create mode 100644 scripts/test_build_backend.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 77553c64fa..a55fdb541f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -812,6 +812,9 @@ jobs: # same failure. Both are only observable in a `pull_request` run, so they # are covered by unit tests instead. - run: uv run --no-project --with pytest pytest scripts/test_check_ecosystem_roundtrip.py + # The build backend decides what the staged `pyproject.toml` says and where + # a dynamic version comes from. Neither is reachable from the Rust tests. + - run: uv run --no-project --with pytest pytest scripts/test_build_backend.py # Lint/format/type-check py-fuzzer # (dogfooding with ty is done in a separate job) - run: uv run --directory=./python/py-fuzzer mypy diff --git a/Cargo.lock b/Cargo.lock index 7232328ef5..bd24958b0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4735,6 +4735,7 @@ dependencies = [ "ty_python_core", "ty_python_semantic", "ty_server", + "ty_site_packages", "ty_static", "walkdir", "wild", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index f834b2b9da..d0dfde8438 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -3528,6 +3528,7 @@ dependencies = [ "rayon", "ruff_db", "ruff_diagnostics", + "ruff_python_ast", "ruff_ranged_value", "ruff_text_size", "salsa", @@ -3545,6 +3546,7 @@ dependencies = [ "ty_project", "ty_python_semantic", "ty_server", + "ty_site_packages", "ty_static", "walkdir", "wild", diff --git a/crates/ty/Cargo.toml b/crates/ty/Cargo.toml index 10fdd4c652..ac11bfa21d 100644 --- a/crates/ty/Cargo.toml +++ b/crates/ty/Cargo.toml @@ -24,6 +24,7 @@ doctest = false [dependencies] ruff_db = { workspace = true, features = ["os", "cache", "junit"] } +ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } ruff_diagnostics = { workspace = true } by_build = { workspace = true } @@ -37,6 +38,7 @@ ty_ide = { workspace = true } ty_project = { workspace = true, features = ["zstd", "junit"] } ty_python_semantic = { workspace = true, features = ["serde"] } ty_server = { workspace = true } +ty_site_packages = { workspace = true } ty_static = { workspace = true } anyhow = { workspace = true } @@ -66,7 +68,6 @@ tikv-jemallocator = { workspace = true } [dev-dependencies] ruff_db = { workspace = true, features = ["testing"] } -ruff_python_ast = { workspace = true } ruff_python_trivia = { workspace = true } ty_module_resolver = { workspace = true } ty_python_core = { workspace = true } diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index ef6ae1a085..75d53e7d98 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -19,7 +19,8 @@ by
by version

Display ty's version

by explain

Explain rules and other parts of ty

by run

Transpile and run a module with python -m <module>

-
by build

Transpile all .by files and write them to out/

+
by init

Start a new project

+
by build

Build the project as python

by compile

Compile .by and .py files to native CPython extension modules

by generate-api-file

Generate an api lockfile (api.lock) summarising the public type-level surface of the project

by transpile

Transpile a file to stdout, or a whole directory in place (reads stdin if no path given)

@@ -247,13 +248,43 @@ by run [OPTIONS] [MODULE] [ARGS]...
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the version of the interpreter that will run it]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--python, --venv path

The interpreter to run on, or the environment holding it.

+

Defaults to the project environment — the same one by check resolves imports against — then $PYTHON, then python3 on PATH.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

[default: default]

+## by init + +Start a new project. + +Writes a `pyproject.toml` that names the basedpython build backend, a `src` layout, and a python version the checker, the transpiler and the interpreter all agree on — so the project is installable, runnable and publishable from the moment it exists. + +

Usage

+ +``` +by init [OPTIONS] [PATH] +``` + +

Arguments

+ +
PATH

Where to create the project [default: the current directory]

+
+ +

Options

+ +
--app

Create an application, with an entry point by run uses. The default

+
--help, -h

Print help (see a summary with '-h')

+
--lib

Create a library: no entry point, the same packaging

+
--name name

The project's name [default: the directory's name]

+
--python-version version

The python version to target [default: the version of the project environment's interpreter]

+
+ ## by build -Transpile all .by files and write them to out/ +Build the project as python. + +The output is the whole project, not only the transpiled half: every `.by` file becomes a `.py`, and every other file — a hand-written `.py`, a `py.typed`, a template, a data file — is carried over to the same place. What the previous build wrote and this one did not is deleted.

Usage

@@ -263,9 +294,12 @@ by build [OPTIONS]

Options

-
--help, -h

Print help

+
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the project's configured python version]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

+
--out, -o dir

Where to write the built project

+

[default: out]

--print-manifest

Report what the build read and produced, as <kind> <value> lines.

+

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

[default: default]

diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index 85ff559aec..0902a8bc7a 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -846,6 +846,154 @@ Defaults to `false`. --- +## `build` + +### `exclude` + +Files to keep out of the build output. + +The syntax is the same as `src.exclude`, and paths are anchored to the +project root. Excluding a `.by` file keeps its transpiled output out of +the build as well. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + exclude = [ + "tests", + "**/*.snapshot", + ] + ``` + +=== "ty.toml" + + ```toml + [build] + exclude = [ + "tests", + "**/*.snapshot", + ] + ``` + +--- + +### `include` + +Files to carry into the build output verbatim, in addition to the ones +that are there by default. + +`by build` mirrors the whole module tree: a `.by` file is transpiled, and +every other file — a hand-written `.py`, a `py.typed` marker, a template, +a data file — is copied to the same place in the output. `include` is for +the files that sit *outside* a module root and still belong in the build, +such as a data directory next to `src`. + +The syntax is the same as `src.include`, and paths are anchored to the +project root. `exclude` takes precedence over `include`. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + include = [ + "assets", + ] + ``` + +=== "ty.toml" + + ```toml + [build] + include = [ + "assets", + ] + ``` + +--- + +### `sources` + +Whether the build output carries the `.by` sources alongside the python +they were transpiled into, with a `by.typed` marker naming them as the +authoritative surface. + +This is what lets one basedpython project depend on another: a downstream +python project reads the transpiled `.py` and is served perfectly, while a +downstream basedpython project reads the `.by` and keeps the declarations +that have no python spelling — `extension` blocks, `raises` clauses, +read-only `let`, sum types. + +Enabled by default. Turn it off to ship python only. + +**Default value**: `true` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + sources = false + ``` + +=== "ty.toml" + + ```toml + [build] + sources = false + ``` + +--- + +### `version-from` + +The module to read `__version__` from, when `[project]` declares +`dynamic = ["version"]`. + +This is read when a wheel or a source distribution is built, not by the +checker: a version has to be settled before the packaging backend sees the +project, and the place it lives is a `.by` module that backend cannot +read. + +The value is a path relative to the project root. + +**Default value**: `null` + +**Type**: `str` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + version-from = "src/app/__init__.by" + ``` + +=== "ty.toml" + + ```toml + [build] + version-from = "src/app/__init__.by" + ``` + +--- + ## `editor` ### `common-aliases` diff --git a/crates/ty/docs/environment.md b/crates/ty/docs/environment.md index 9a0b1f9d13..54bead8781 100644 --- a/crates/ty/docs/environment.md +++ b/crates/ty/docs/environment.md @@ -59,6 +59,13 @@ Used to determine the name of the active Conda environment. Used to detect the path of an active Conda environment. If both `VIRTUAL_ENV` and `CONDA_PREFIX` are present, `VIRTUAL_ENV` will be preferred. +### `PYTHON` + +The interpreter `by run` executes the transpiled program on. + +Overrides the project environment, and is itself overridden by +`by run --python`. + ### `PYTHONPATH` Adds additional directories to ty's search paths. diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 3c9cfae9d3..1ac92d09c3 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -81,6 +81,12 @@ pub(crate) enum Command { /// [default: the version of the interpreter that will run it] #[arg(long, value_name = "VERSION")] min_version: Option, + /// The interpreter to run on, or the environment holding it. + /// + /// Defaults to the project environment — the same one `by check` + /// resolves imports against — then `$PYTHON`, then `python3` on `PATH`. + #[arg(long, value_name = "PATH", alias = "venv")] + python: Option, #[command(flatten)] lowering: LoweringArgs, /// Compile every imported module to a native extension first. @@ -95,12 +101,52 @@ pub(crate) enum Command { compiled: bool, }, - /// Transpile all .by files and write them to out/. + /// Start a new project. + /// + /// Writes a `pyproject.toml` that names the basedpython build backend, a + /// `src` layout, and a python version the checker, the transpiler and the + /// interpreter all agree on — so the project is installable, runnable and + /// publishable from the moment it exists. + Init { + /// Where to create the project [default: the current directory] + #[arg(value_name = "PATH")] + path: Option, + /// The project's name [default: the directory's name] + #[arg(long, value_name = "NAME")] + name: Option, + /// Create a library: no entry point, the same packaging. + #[arg(long, conflicts_with = "app")] + lib: bool, + /// Create an application, with an entry point `by run` uses. The default. + #[arg(long)] + app: bool, + /// The python version to target + /// [default: the version of the project environment's interpreter] + #[arg(long, value_name = "VERSION")] + python_version: Option, + }, + + /// Build the project as python. + /// + /// The output is the whole project, not only the transpiled half: every `.by` + /// file becomes a `.py`, and every other file — a hand-written `.py`, a + /// `py.typed`, a template, a data file — is carried over to the same place. + /// What the previous build wrote and this one did not is deleted. Build { /// minimum Python version the output must run on /// [default: the project's configured python version] #[arg(long, value_name = "VERSION")] min_version: Option, + /// Where to write the built project. + #[arg(short = 'o', long, value_name = "DIR", default_value = "out")] + out: PathBuf, + /// Report what the build read and produced, as ` ` lines. + /// + /// `input ` for every file the project is made of — what a source + /// distribution has to carry to rebuild into the same thing — and + /// `package ` for every top-level package that came out. + #[arg(long)] + print_manifest: bool, #[command(flatten)] lowering: LoweringArgs, }, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 2f15a62359..5dcd337348 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeSet, HashMap}; +use std::collections::HashMap; use std::ffi::OsStr; use std::fs; use std::io::{self, Read}; @@ -17,10 +17,13 @@ use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf}; use ruff_text_size::TextRange; use sha2::{Digest, Sha256}; use ty_project::{Db, ProjectDatabase, ProjectMetadata}; +use ty_site_packages::{PythonEnvironment, SysPrefixPathOrigin}; +use ty_static::EnvVars; use walkdir::WalkDir; use crate::ExitStatus; use crate::args::LoweringArgs; +use crate::by_staging::{Staging, relative_destination, transpiled_destination}; /// The python version the emitted code must run on when `--min-version` is not /// given: the one the project configures (`environment.python-version`, else the @@ -122,8 +125,11 @@ pub(crate) fn cmd_run( min_version: Option<&str>, lowering: &LoweringArgs, compiled: bool, + python_flag: Option<&Path>, ) -> anyhow::Result { - let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let cwd = std::env::current_dir().context("failed to get current directory")?; + let interpreter = discover_interpreter(python_flag, &cwd); + let python = interpreter.path.clone(); // `run` executes on a specific interpreter, so by default target *its* // version: the emitted code (dataclass `slots=`, PEP 695 syntax, …) must // match what that python actually supports. an explicit `--min-version` @@ -133,13 +139,13 @@ pub(crate) fn cmd_run( let mut config = match (min_version, probed) { (Some(flag), probed) => { let config = parse_version(flag)?; - if let Some(interpreter) = probed - && config.min_version > interpreter + if let Some(found) = probed + && config.min_version > found { anyhow::bail!( - "--min-version {flag} is newer than `{python}` ({interpreter}), \ - which could not run the emitted code — \ - set PYTHON to a {flag}+ interpreter" + "--min-version {flag} is newer than the interpreter this would run on: \ + `{python}` is {found}, from {}", + interpreter.origin ); } config @@ -150,11 +156,26 @@ pub(crate) fn cmd_run( }, (None, None) => Config::default(), }; + // a program written for the python the project declares cannot be run by an + // older one: the source itself may use syntax that python has no lowering + // for (`match`, for one), and the failure lands as a `SyntaxError` inside + // generated code rather than as anything the author can act on + if let Some(found) = probed { + let configured = configured_min_version(&cwd); + if min_version.is_none() && found < configured { + anyhow::bail!( + "this project targets python {configured}, but the interpreter this would run on \ + is {found}: `{python}`, from {}\n \ + use an interpreter that is {configured} or newer, or pass \ + `--min-version {found}` to build for this one", + interpreter.origin + ); + } + } lowering.apply(&mut config)?; - let cwd = std::env::current_dir().context("failed to get current directory")?; let tmp = tempfile::TempDir::new().context("failed to create temp directory")?; - let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES)?; + let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, None)?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Failure); @@ -178,6 +199,7 @@ pub(crate) fn cmd_run( // each generated `.py` paired with its source `.by` and the line table that // lifts generated line numbers back to `.by` lines (for traceback rewriting) let mut traceback_entries: Vec = Vec::new(); + let mut staging = Staging::new(tmp.path()); let ok = render_check_and_transpile( &db, &handles, @@ -185,16 +207,21 @@ pub(crate) fn cmd_run( CheckGate::AllErrors, &rebuilder, |emitted| { - let py = tmp - .path() - .join(module_relative_path(&roots, &root, emitted.by_path)); - traceback_entries.push(write_module(py, emitted)?); + let relative = transpiled_destination(&roots, &root, emitted.by_path); + traceback_entries.push(stage_module(&mut staging, &relative, emitted)?); Ok(()) }, )?; if !ok { return Ok(ExitStatus::Failure); } + // a program is its data as much as its modules: a `.py` module it imports, a + // json file it opens, a template it renders. running out of a directory + // holding only the transpiled half fails on the first of them + stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_by_typed_markers(&db, &mut staging, &roots, &root)?; + write_traceback_runtime(&mut staging, &traceback_entries)?; + staging.finish()?; if compiled { // the extension lands beside the generated `.py`, and python's finder @@ -258,15 +285,18 @@ pub(crate) fn cmd_run( } } - write_traceback_runtime(tmp.path(), &traceback_entries)?; - let status = Command::new(&python) .arg(BY_RUNNER_FILENAME) .arg(&module) .args(args) .current_dir(tmp.path()) .status() - .with_context(|| format!("{python}: failed to execute"))?; + .with_context(|| { + format!( + "could not run `{python}`, the interpreter from {}", + interpreter.origin + ) + })?; let code = status.code().unwrap_or(1); // drop the temp dir explicitly: `process::exit` skips destructors, so @@ -293,37 +323,162 @@ fn module_roots(db: &ProjectDatabase, cwd: &Path) -> Vec { roots } -/// Where `bpy`'s transpiled python goes, relative to the output root. +/// The source roots a distribution's packages come from. /// -/// The tree mirrored is the *module* tree, not the directory tree: a src-layout -/// project's `src/pkg/main.by` is the module `pkg.main`, so it has to land at -/// `pkg/main.py`. Mirroring the directory instead emits `src/pkg/main.py`, -/// whose module is `src.pkg.main` — a name nothing imports, and one `run.main` -/// cannot sensibly be set to. -fn module_relative_path(roots: &[PathBuf], root: &Path, bpy: &Path) -> PathBuf { - let relative = roots +/// The project root is always a module root — it is what lets `tests/` and a +/// script beside it resolve their imports — but for a src-layout project it is +/// not where the *distribution* lives. `src/app` is a package of this project; +/// `tests` beside it is not something anyone installs. So when the project +/// declares somewhere for its modules to live, that is where they live, and the +/// root counts only when it is the only answer. +fn packaging_roots(roots: &[PathBuf], root: &Path) -> Vec { + let declared: Vec = roots .iter() - .find_map(|candidate| bpy.strip_prefix(candidate).ok()) - .or_else(|| bpy.strip_prefix(root).ok()) - .unwrap_or(bpy); - // whatever happened above, the result has to be *relative*: joined onto the - // output directory an absolute path replaces it outright, so every emitted - // file would land outside the output tree. keeping only the named components - // also drops any `..`, which would climb back out of it - relative - .components() - .filter_map(|component| match component { - std::path::Component::Normal(name) => Some(name), - _ => None, + .filter(|candidate| candidate.as_path() != root) + .cloned() + .collect(); + if declared.is_empty() { + vec![root.to_path_buf()] + } else { + declared + } +} + +/// The top-level packages the build produced, as a distribution would ship them. +fn staged_packages(staging: &Staging, roots: &[PathBuf], root: &Path) -> Vec { + let packaging = packaging_roots(roots, root); + let mut packages: Vec = staging + .entries() + .filter(|(_, source)| { + source.is_some_and(|source| { + packaging + .iter() + .any(|candidate| source.starts_with(candidate)) + }) + }) + .filter_map(|(destination, _)| { + let mut components = destination.components(); + let package = components.next()?; + let rest: PathBuf = components.collect(); + matches!( + rest.to_str(), + Some("__init__.py" | "__init__.pyi" | "__init__.by" | "__init__.byi") + ) + .then(|| package.as_os_str().to_str().map(str::to_owned)) + .flatten() }) - .collect::() - .with_extension("py") + .collect(); + packages.sort(); + packages.dedup(); + packages +} + +/// Carry every file the transpiler did not produce into the output tree. +/// +/// This is what makes the output a project rather than a heap of transpiled +/// modules: a hand-written `.py` sibling, a `py.typed`, a template, a fixture — +/// all of it lands in the same relative place, so the output imports and reads +/// data exactly the way the source tree does. +fn stage_verbatim( + db: &ProjectDatabase, + root: &Path, + roots: &[PathBuf], + staging: &mut Staging, +) -> anyhow::Result<()> { + let settings = db.project().settings(db); + let build = settings.build(); + // a file the project excludes from itself is not part of what it ships, so + // `src.exclude` bounds the build and `build.exclude` narrows it further + let src = settings.src(); + let out = staging.out().to_path_buf(); + + for entry in WalkDir::new(root) + .into_iter() + .filter_entry(|entry| { + // the output tree is not an input to itself, wherever `--out` put it + if entry.path() == out { + return false; + } + if !may_hold_build_content(entry) { + return false; + } + !entry.file_type().is_dir() + || SystemPath::from_std_path(entry.path()).is_none_or(|path| { + build.is_directory_included(path) && src.is_directory_included(path) + }) + }) + .filter_map(Result::ok) + { + let path = entry.path(); + if entry.path_is_symlink() || !entry.file_type().is_file() { + continue; + } + let extension = path.extension().and_then(OsStr::to_str); + // a `.by` is an input, and it is carried over only to be read by a + // downstream basedpython project — never for python to import + if matches!(extension, Some("by" | "byi")) && !build.sources() { + continue; + } + let Some(system_path) = SystemPath::from_std_path(path) else { + continue; + }; + if !build.is_file_included(system_path) || !src.is_file_included(system_path) { + continue; + } + staging.copy(&relative_destination(roots, root, path), path)?; + } + Ok(()) +} + +/// Write the `by.typed` marker into every package the build ships. +/// +/// The marker says two things to a project that installs this one, and both are +/// things nothing else can tell it. Its presence says the `.by` beside a module is +/// the authoritative surface, to be read in preference to the python it was +/// transpiled into — the same bargain `py.typed` strikes for inline annotations. +/// Its contents say which of this project's dependencies are part of its own +/// interface, which a `pyproject.toml` cannot, because nothing installs one. +/// +/// Only the packages the project *ships* are marked, and they are read off what +/// was written rather than off the source layout — a `tests` package beside `src` +/// is neither shipped nor anybody else's business. +#[allow(clippy::print_stderr)] +fn stage_by_typed_markers( + db: &ProjectDatabase, + staging: &mut Staging, + roots: &[PathBuf], + root: &Path, +) -> anyhow::Result<()> { + let exported = db + .project() + .settings(db) + .analysis() + .exported_dependencies + .clone() + .unwrap_or_default(); + // written whether or not the `.by` sources went with it: the precedence claim + // is vacuous without them — nothing to prefer — but the export declaration is + // not, and a python-only build still has dependencies it hands out on purpose + let marker = ty_module_resolver::Marker::render(&exported); + + for package in staged_packages(staging, roots, root) { + staging.write( + &Path::new(&package).join(ty_module_resolver::BY_TYPED), + None, + &marker, + )?; + } + + if !exported.is_empty() { + eprintln!("exporting {}", exported.join(", ")); + } + Ok(()) } /// The dotted module name a file laid out at `relative` will be imported under. /// /// The tree the generated python is written into *is* the module tree — every -/// file lands at [`module_relative_path`] — so the name is that path with its +/// file lands at [`transpiled_destination`] — so the name is that path with its /// separators turned into dots. `pkg/__init__.py` is the package `pkg` itself, /// which is the name a class defined in it reports as its `__module__`. /// @@ -410,6 +565,106 @@ fn configured_main(db: &ProjectDatabase) -> Option { Some((**main).clone()) } +/// The python version a brand new project should target. +/// +/// The environment it will be developed in is the right answer when there is +/// one: the version the checker targets, the version the transpiler emits for, +/// and the version that runs the result should be one version rather than three. +/// A project being created usually has no environment yet, though, and the bare +/// `python3` that turns up on `PATH` instead is whatever the operating system +/// shipped years ago. Pinning a new project to that is how a project ends up +/// targeting 3.9 for its whole life without anyone choosing to. +pub(crate) fn default_project_python_version() -> PythonVersion { + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let interpreter = discover_interpreter(None, &cwd); + if interpreter.is_from_path { + return ruff_python_ast::PythonVersion::latest() + .to_string() + .parse() + .unwrap_or_else(|_| Config::default().min_version); + } + detect_python_version(&interpreter.path).unwrap_or_else(|| Config::default().min_version) +} + +/// The interpreter `by run` executes on, and how it was chosen. +struct Interpreter { + path: String, + origin: String, + /// whether this is the bare `python3` off `PATH` — the last resort, and the + /// only origin that says nothing about what the project targets + is_from_path: bool, +} + +/// Find the interpreter to run the program on. +/// +/// The project environment comes first, because that is the environment the +/// project *is*: `by check` resolved this project's imports against it, so +/// running against a different python answers a question nobody asked. An +/// explicit choice still wins over discovery — `--python` for this one run, +/// `$PYTHON` for a shell that has already decided — and a bare `python3` off +/// `PATH` is the last resort rather than the first. +fn discover_interpreter(flag: Option<&Path>, root: &Path) -> Interpreter { + let named = |path: String, origin: &str| Interpreter { + path, + origin: origin.to_owned(), + is_from_path: false, + }; + + if let Some(flag) = flag { + // a `--python` may name the interpreter itself or the environment it + // lives in, the same way `by check --python` does + if flag.is_file() { + return named(flag.display().to_string(), "`--python`"); + } + if let Some(interpreter) = + interpreter_in_environment(flag, SysPrefixPathOrigin::PythonCliFlag) + { + return named(interpreter, "`--python`"); + } + return named(flag.display().to_string(), "`--python`"); + } + + if let Ok(python) = std::env::var(EnvVars::PYTHON) { + return named(python, "`PYTHON`"); + } + + if let Some(sys_root) = SystemPath::from_std_path(root) { + let system = OsSystem::new(sys_root); + if let Ok(Some(environment)) = PythonEnvironment::discover(sys_root, &system) + && let Some(interpreter) = environment.interpreter(&system) + { + // discovery ends by falling back to whatever python is on `PATH`, + // which is an interpreter but not a *project* environment — the + // difference matters to anything asking what this project targets + let is_from_path = matches!( + environment.origin(), + SysPrefixPathOrigin::PythonBinary | SysPrefixPathOrigin::SelfEnvironment + ); + return Interpreter { + path: interpreter.to_string(), + origin: environment.origin().to_string(), + is_from_path, + }; + } + } + + Interpreter { + path: "python3".to_owned(), + origin: "`PATH`".to_owned(), + is_from_path: true, + } +} + +/// The interpreter inside the environment rooted at `path`, if there is one. +fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Option { + let sys_path = SystemPath::from_std_path(path)?; + let system = OsSystem::new(sys_path); + let environment = PythonEnvironment::new(sys_path, origin, &system).ok()?; + environment + .interpreter(&system) + .map(|interpreter| interpreter.to_string()) +} + /// Probe `python`'s `major.minor` version (e.g. `3.9`) so `run` can target the /// interpreter it will execute on. Returns `None` if the interpreter can't be /// run or its output can't be parsed. @@ -431,20 +686,30 @@ fn detect_python_version(python: &str) -> Option { pub(crate) fn cmd_build( min_version: Option<&str>, lowering: &LoweringArgs, + out: &Path, + print_manifest: bool, ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; let mut config = version_config(min_version, &cwd)?; lowering.apply(&mut config)?; - let out = cwd.join("out"); - let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES)?; + // the output directory is settled before the project is read, because it is + // the one directory the project must not be read *from*: it holds a copy of + // every source this build is about to write. canonical, because that is what + // the paths it is compared against are — creating it first is what makes + // canonicalizing it possible + let out = cwd.join(out); + fs::create_dir_all(&out).with_context(|| format!("could not create {}", out.display()))?; + let out = fs::canonicalize(&out).unwrap_or(out); + + let (db, handles, rebuilder, root) = build_project_db(&cwd, BY_SOURCES, Some(&out))?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Success); } let file_count = handles.len(); let roots = module_roots(&db, &root); - let mut packages: BTreeSet = BTreeSet::new(); + let mut staging = Staging::new(&out); // `out/` outlives the build that wrote it — it is what a test runner, a // debugger or an editor plugin sees — so the sourcemap goes with it. this is // the directory where a `.by` really can be saved after the transpile, which @@ -457,14 +722,8 @@ pub(crate) fn cmd_build( CheckGate::ParseErrorsOnly, &rebuilder, |emitted| { - let relative = module_relative_path(&roots, &root, emitted.by_path); - if relative.components().count() > 1 - && let Some(package) = relative.components().next() - { - packages.insert(out.join(package)); - } - - let entry = write_module(out.join(relative), emitted)?; + let relative = transpiled_destination(&roots, &root, emitted.by_path); + let entry = stage_module(&mut staging, &relative, emitted)?; eprintln!( "{} -> {}", emitted.by_path.display(), @@ -476,41 +735,38 @@ pub(crate) fn cmd_build( )? { return Ok(ExitStatus::Failure); } - write_sourcemap_module(&out, &entries)?; - write_markers(&db, &packages)?; + stage_verbatim(&db, &root, &roots, &mut staging)?; + stage_by_typed_markers(&db, &mut staging, &roots, &root)?; + write_sourcemap_module(&mut staging, &entries)?; + if print_manifest { + print_build_manifest(&staging, &roots, &root)?; + } + staging.finish()?; eprintln!("\nbuild complete ({file_count} files)"); Ok(ExitStatus::Success) } -/// Writes the `by.typed` marker into every package the build emitted. +/// Report what the build read and what it produced, as ` ` lines. /// -/// The marker is what tells a project that installs this one that its packages -/// are basedpython's, and it carries the one thing a `pyproject.toml` cannot tell -/// them: which of this project's dependencies are part of its own interface. -/// Nothing installs a `pyproject.toml`, and this rides along inside the package. -#[allow(clippy::print_stderr)] -fn write_markers(db: &ProjectDatabase, packages: &BTreeSet) -> anyhow::Result<()> { - let exported = db - .project() - .settings(db) - .analysis() - .exported_dependencies - .clone() - .unwrap_or_default(); - let marker = ty_module_resolver::Marker::render(&exported); +/// Two questions, both of which only the build can answer: which files this +/// project is made of — a source distribution has to carry exactly those, since +/// they are what rebuilds into the same wheel — and which top-level packages came +/// out. Answering them here rather than in the packaging layer keeps one answer +/// to "what is this project", instead of a second one that has to be kept in step. +#[allow(clippy::print_stdout)] +fn print_build_manifest(staging: &Staging, roots: &[PathBuf], root: &Path) -> anyhow::Result<()> { + use std::io::Write as _; - for package in packages { - let path = package.join(ty_module_resolver::BY_TYPED); - fs::create_dir_all(package)?; - fs::write(&path, &marker)?; + let mut stdout = io::stdout().lock(); + for input in staging.inputs() { + let relative = input.strip_prefix(root).unwrap_or(input); + writeln!(stdout, "input {}", relative.display())?; } - - if !exported.is_empty() { - eprintln!("exporting {}", exported.join(", ")); + for package in staged_packages(staging, roots, root) { + writeln!(stdout, "package {package}")?; } - Ok(()) } @@ -577,7 +833,7 @@ pub(crate) fn cmd_compile( // gradual, and `--no-any` would then fail on noise // `compile` embeds fallback source produced by the untyped transpile, which // takes no db, so the rebuilder the other commands thread through is unused here - let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES, None)?; // the database holds the whole project so a type imported from a sibling // resolves, but only the files that were *asked for* are checked and emitted. @@ -874,6 +1130,11 @@ const NON_SOURCE_DIRS: &[&str] = &[ "dist", "node_modules", "out", + // rust's build directory, which a basedpython project has whenever it also + // has an extension crate — and which the build would otherwise copy in full. + // a project that really does have a package called `target` can take it back + // with `exclude = ["!target"]` + "target", ]; fn cmd_transpile_dir(dir: &Path, reverse: bool, config: &Config) -> anyhow::Result { @@ -985,7 +1246,7 @@ fn reverse_dir_converting( /// build`, but written in place rather than to `out/`). #[allow(clippy::print_stderr)] fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { - let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES)?; + let (db, handles, rebuilder, _root) = build_project_db(dir, BY_SOURCES, None)?; if handles.is_empty() { eprintln!("no .by files found"); return Ok(ExitStatus::Success); @@ -1028,6 +1289,24 @@ fn py_source_files(root: &Path) -> Vec { .collect() } +/// Whether the build walk may descend into this entry. +/// +/// Narrower than [`may_contain_sources`] on purpose, for the same reason +/// [`is_hidden_within`] is: this walk applies the project's own `src` and `build` +/// filters as it goes, so everything ty's `src.exclude` defaults already drop is +/// covered — and re-dropping it here would take back a file that a negated +/// exclude deliberately re-included. Only the directories ty's defaults *leave* +/// (and hidden ones) still have to be turned away. +fn may_hold_build_content(entry: &walkdir::DirEntry) -> bool { + if entry.depth() == 0 || !entry.file_type().is_dir() { + return true; + } + entry + .file_name() + .to_str() + .is_some_and(|name| !name.starts_with('.') && !NON_SOURCE_DIRS_TY_ALLOWS.contains(&name)) +} + /// Whether a project walk may descend into this entry: hidden directories /// (`.claude`, `.git`, `.venv`, …) and [`NON_SOURCE_DIRS`] never hold /// first-party source. The walk root itself is always entered, even when the @@ -1065,17 +1344,20 @@ struct TracebackEntry { py_digest: String, } -/// write one emitted module out and describe it, in that order: the digests -/// are over the bytes that just landed on disk -fn write_module(py_path: PathBuf, emitted: &Transpiled<'_>) -> anyhow::Result { - if let Some(parent) = py_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&py_path, emitted.python) - .with_context(|| format!("failed to write {}", py_path.display()))?; +/// stage one emitted module and describe it, in that order: the digests are over +/// the bytes that just landed on disk +/// +/// it goes through the staging rather than straight to the path, so that a module +/// two sources both claim is reported, and so that the file is one the manifest +/// knows about and a later build can clean up +fn stage_module( + staging: &mut Staging, + relative: &Path, + emitted: &Transpiled<'_>, +) -> anyhow::Result { + staging.write(relative, Some(emitted.by_path), emitted.python)?; Ok(TracebackEntry { - py_path, + py_path: staging.out().join(relative), by_path: fs::canonicalize(emitted.by_path) .unwrap_or_else(|_| emitted.by_path.to_path_buf()), line_map: emitted.line_map.to_vec(), @@ -1100,7 +1382,7 @@ fn content_digest(bytes: &[u8]) -> String { /// both tables are keyed by the generated path exactly as written here. a /// consumer that normalises those keys — the runner shim resolves symlinks, for /// one — has to keep the original key to reach the entry's digests -fn write_sourcemap_module(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Result<()> { +fn write_sourcemap_module(staging: &mut Staging, entries: &[TracebackEntry]) -> anyhow::Result<()> { use std::fmt::Write as _; let mut map_src = String::from( @@ -1146,20 +1428,21 @@ fn write_sourcemap_module(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Res } map_src.push_str("}\n"); - fs::create_dir_all(dir).with_context(|| format!("failed to create {}", dir.display()))?; - fs::write(dir.join(BY_SOURCEMAP_FILENAME), map_src) - .with_context(|| "failed to write sourcemap module")?; - Ok(()) + staging.write(Path::new(BY_SOURCEMAP_FILENAME), None, &map_src) } /// write the sourcemap module + runner shim into the run dir. the shim runs the /// target module and, on an uncaught exception, rewrites traceback frames in /// generated files back to their `.by` source location -fn write_traceback_runtime(dir: &Path, entries: &[TracebackEntry]) -> anyhow::Result<()> { - write_sourcemap_module(dir, entries)?; - fs::write(dir.join(BY_RUNNER_FILENAME), BY_RUNNER_SRC) - .with_context(|| "failed to write runner shim")?; - Ok(()) +fn write_traceback_runtime( + staging: &mut Staging, + entries: &[TracebackEntry], +) -> anyhow::Result<()> { + write_sourcemap_module(staging, entries)?; + // through the staging, so that a project of its own with this name is + // reported as the collision it is rather than silently overwritten by a shim + // it knows nothing about + staging.write(Path::new(BY_RUNNER_FILENAME), None, BY_RUNNER_SRC) } /// Render a string as a python string literal (double-quoted, minimal escaping). @@ -1382,6 +1665,7 @@ const NON_SOURCE_DIRS_TY_ALLOWS: &[&str] = &[ ".pytest_cache", "build", "out", + "target", ]; /// Whether `path` sits inside a hidden or build-output directory under `root`. @@ -1402,7 +1686,11 @@ fn is_hidden_within(path: &Path, root: &Path) -> bool { /// extension is in `extensions` /// — the same set `by check` walks, so `src.exclude` and the ignore files it /// honours apply here too — and the means to build the same project again. -fn build_project_db(cwd: &Path, extensions: &[&str]) -> anyhow::Result { +fn build_project_db( + cwd: &Path, + extensions: &[&str], + output: Option<&Path>, +) -> anyhow::Result { // the project root must be canonicalized the same way the included files // are (below) so it stays a path *prefix* of them: otherwise a file's // search path isn't recognized as first-party and boundary diagnostics @@ -1439,6 +1727,10 @@ fn build_project_db(cwd: &Path, extensions: &[&str]) -> anyhow::Result ExitStat mod tests { use super::{ BY_SOURCEMAP_FILENAME, TracebackEntry, content_digest, dotted_module_name, - is_hidden_within, module_relative_path, reverse_dir, reverse_dir_converting, - write_sourcemap_module, + is_hidden_within, reverse_dir, reverse_dir_converting, write_sourcemap_module, }; use crate::ExitStatus; use by_transforms::config::Config; use std::path::{Path, PathBuf}; - /// the output tree mirrors the module tree, so a src-layout project's source - /// root is stripped rather than mirrored - #[test] - fn a_module_root_is_stripped() { - let roots = vec![PathBuf::from("/p/src"), PathBuf::from("/p")]; - assert_eq!( - module_relative_path(&roots, Path::new("/p"), Path::new("/p/src/pkg/main.by")), - PathBuf::from("pkg/main.py") - ); - } - - /// the deepest root wins: a file under `src` is `pkg.main`, not `src.pkg.main` - #[test] - fn the_deepest_root_wins() { - let roots = vec![PathBuf::from("/p/src"), PathBuf::from("/p")]; - assert_eq!( - module_relative_path(&roots, Path::new("/p"), Path::new("/p/top.by")), - PathBuf::from("top.py") - ); - } - /// a compiled module carries its name into every type it emits, and cpython /// reads a class's `__module__` off the front of that — so the name of a file /// inside the tree is the whole path to it, not the file's own stem @@ -1712,22 +1982,6 @@ mod tests { assert_eq!(dotted_module_name(Path::new("__init__.py")), None); } - /// a root that shares no prefix with the file — which is what `canonicalize` - /// and `current_dir` disagreeing produced on windows — must not leave the - /// path absolute: joining that onto the output directory discards the output - /// directory entirely, so every emitted file lands outside it - #[test] - fn a_root_that_does_not_match_still_yields_a_relative_path() { - let unrelated = vec![PathBuf::from("/other/src")]; - let emitted = - module_relative_path(&unrelated, Path::new("/other"), Path::new("/p/pkg/main.by")); - assert!( - emitted.is_relative(), - "an absolute result escapes the output directory: {}", - emitted.display() - ); - } - #[test] fn a_hidden_directory_is_not_project_source() { let root = Path::new("/p"); @@ -1867,7 +2121,8 @@ mod tests { py_digest: content_digest(b"the generated python"), }]; - write_sourcemap_module(dir.path(), &entries)?; + let mut staging = crate::by_staging::Staging::new(dir.path()); + write_sourcemap_module(&mut staging, &entries)?; let emitted = std::fs::read_to_string(dir.path().join(BY_SOURCEMAP_FILENAME))?; assert!( diff --git a/crates/ty/src/by_init.rs b/crates/ty/src/by_init.rs new file mode 100644 index 0000000000..eb92f319c2 --- /dev/null +++ b/crates/ty/src/by_init.rs @@ -0,0 +1,207 @@ +//! Starting a project. +//! +//! Everything a basedpython project needs to be installable, checkable and +//! publishable is decided in its `pyproject.toml`, and every one of those +//! decisions is one somebody has to get right before writing a line of code: the +//! build backend, the layout the module tree is read from, the python version the +//! checker and the transpiler both target. `by init` writes them consistently, so +//! that the answer to "how do I ship this" is settled at the point the project is +//! created rather than discovered afterwards. + +use std::fs; +use std::path::Path; + +use anyhow::Context; + +use crate::ExitStatus; + +/// What kind of project is being started. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProjectKind { + /// something to run: it gets an entry point, and `by run` alone will run it + Application, + /// something to import: no entry point, but the same packaging + Library, +} + +#[allow(clippy::print_stderr)] +pub(crate) fn cmd_init( + path: Option<&Path>, + name: Option<&str>, + kind: ProjectKind, + python_version: &str, +) -> anyhow::Result { + let cwd = std::env::current_dir().context("failed to get current directory")?; + let root = match path { + Some(path) if path.is_absolute() => path.to_path_buf(), + Some(path) => cwd.join(path), + None => cwd, + }; + + let project_name = match name { + Some(name) => name.to_owned(), + None => directory_name(&root)?, + }; + let package = package_name(&project_name); + + let pyproject = root.join("pyproject.toml"); + if pyproject.exists() { + anyhow::bail!( + "`{}` already exists — `by init` will not write over a project that is already there", + pyproject.display() + ); + } + + let package_root = root.join("src").join(&package); + fs::create_dir_all(&package_root) + .with_context(|| format!("could not create {}", package_root.display()))?; + + write_new( + &pyproject, + &render_pyproject(&project_name, &package, kind, python_version), + )?; + write_new( + &root.join(".python-version"), + &format!("{python_version}\n"), + )?; + write_new(&root.join("README.md"), &format!("# {project_name}\n"))?; + write_new(&package_root.join("__init__.by"), "")?; + if kind == ProjectKind::Application { + write_new(&package_root.join("main.by"), MAIN)?; + } + + eprintln!("initialized `{project_name}` at {}", root.display()); + if kind == ProjectKind::Application { + eprintln!("run it with `by run`"); + } + eprintln!("build a wheel with `uv build`"); + Ok(ExitStatus::Success) +} + +/// The entry module of a new application. +/// +/// `main` taking no arguments is the smallest thing that is still a real entry +/// point: give it parameters and they become command-line arguments. +const MAIN: &str = "\ +def main(): + print(\"hello from basedpython\") + + +main() +"; + +fn render_pyproject( + project_name: &str, + package: &str, + kind: ProjectKind, + python_version: &str, +) -> String { + // the backend a new project builds with needs a floor: without one, a future + // release that changes how a project is built would change how *this* project + // is built, without the project having said anything + let backend_version = env!("CARGO_PKG_VERSION"); + let entry_point = match kind { + ProjectKind::Application => { + format!("\n[tool.basedpython.run]\nmain = \"{package}.main\"\n") + } + ProjectKind::Library => String::new(), + }; + format!( + "\ +[build-system] +requires = [\"basedpython>={backend_version}\"] +build-backend = \"basedpython.build\" + +[project] +name = \"{project_name}\" +version = \"0.1.0\" +description = \"\" +readme = \"README.md\" +requires-python = \">={python_version}\" +dependencies = [] +{entry_point}" + ) +} + +/// The importable name for a project called `project_name`. +/// +/// A distribution name may hold `-` and `.`, which no module name can, so the +/// package directory is the normalized form — the same one every python packaging +/// tool arrives at. +fn package_name(project_name: &str) -> String { + project_name + .chars() + .map(|character| match character { + '-' | '.' => '_', + other => other.to_ascii_lowercase(), + }) + .collect() +} + +fn directory_name(root: &Path) -> anyhow::Result { + root.file_name() + .and_then(std::ffi::OsStr::to_str) + .map(str::to_owned) + .with_context(|| { + format!( + "could not read a project name from `{}` — pass one with `--name`", + root.display() + ) + }) +} + +/// Write a file, leaving anything already there alone. +/// +/// `by init` refuses to start on top of an existing project, but a directory can +/// still hold a `README.md` somebody wrote. Nothing here is worth overwriting it +/// for. +fn write_new(path: &Path, contents: &str) -> anyhow::Result<()> { + if path.exists() { + return Ok(()); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("could not create {}", parent.display()))?; + } + fs::write(path, contents).with_context(|| format!("could not write {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_distribution_name_becomes_an_importable_package_name() { + assert_eq!(package_name("my-project"), "my_project"); + assert_eq!(package_name("My.Project"), "my_project"); + assert_eq!(package_name("plain"), "plain"); + } + + #[test] + fn an_application_gets_an_entry_point_and_a_library_does_not() { + let application = render_pyproject("app", "app", ProjectKind::Application, "3.13"); + assert!(application.contains("[tool.basedpython.run]")); + assert!(application.contains("main = \"app.main\"")); + + let library = render_pyproject("lib", "lib", ProjectKind::Library, "3.13"); + assert!(!library.contains("[tool.basedpython.run]")); + } + + /// what is written has to be what the packaging path reads: the backend, and + /// a version floor the checker and the transpiler both target + #[test] + fn a_new_project_is_installable_as_written() { + let rendered = render_pyproject("thing", "thing", ProjectKind::Library, "3.12"); + assert!(rendered.contains("build-backend = \"basedpython.build\"")); + // with a floor, so that a later release cannot change how a project + // written today is built + assert!( + rendered.contains(&format!( + "requires = [\"basedpython>={}\"]", + env!("CARGO_PKG_VERSION") + )), + "{rendered}" + ); + assert!(rendered.contains("requires-python = \">=3.12\"")); + } +} diff --git a/crates/ty/src/by_staging.rs b/crates/ty/src/by_staging.rs new file mode 100644 index 0000000000..3d9b2cbedf --- /dev/null +++ b/crates/ty/src/by_staging.rs @@ -0,0 +1,402 @@ +//! Writing a project out as python. +//! +//! `by build` and `by run` both need the same thing: the project, rendered as a +//! directory python can import. That is more than the transpiled `.by` files. A +//! project is also its hand-written `.py` modules, its `py.typed` marker, its +//! templates and json and fixture data — and a tree holding only the transpiled +//! half is not a project at all. A module that imports a `.py` sibling fails to +//! import, and anything that opens a data file relative to the working directory +//! fails to open it. +//! +//! So the output tree mirrors the project: every file is carried over to the same +//! relative place, `.by` sources being the ones that change on the way (they are +//! transpiled, and, when `build.sources` is on, carried over as well so a +//! downstream basedpython project can read them). The one rearrangement is the +//! module roots: a src-layout project's `src/pkg/a.by` is the module `pkg.a`, so +//! it lands at `pkg/a.py`, not `src/pkg/a.py`. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::Context; + +/// The name of the file that records what the last build wrote. +/// +/// A build that only ever adds files leaves the output tree accumulating modules +/// that were deleted from the source months ago. They keep importing, so nothing +/// locally ever notices — until a wheel built from the same tree ships them, or +/// until one shadows a module that moved. The manifest is what makes the output a +/// mirror rather than a pile: what the previous build wrote and this one did not +/// is deleted. +pub(crate) const MANIFEST_FILENAME: &str = ".by-manifest"; + +/// An output tree being written. +pub(crate) struct Staging { + out: PathBuf, + /// relative destination -> the source it came from, for collision reporting + written: BTreeMap>, +} + +impl Staging { + pub(crate) fn new(out: &Path) -> Self { + Self { + out: out.to_path_buf(), + written: BTreeMap::new(), + } + } + + pub(crate) fn out(&self) -> &Path { + &self.out + } + + /// Every file the build read, in sorted order and without duplicates. + /// + /// A `.by` appears once even though it produced two outputs — the python it + /// was transpiled into and the copy of itself carried alongside. + pub(crate) fn inputs(&self) -> BTreeSet<&Path> { + self.written + .values() + .filter_map(|source| source.as_deref()) + .collect() + } + + /// Every written path paired with the file it came from. + pub(crate) fn entries(&self) -> impl Iterator)> { + self.written + .iter() + .map(|(destination, source)| (destination.as_path(), source.as_deref())) + } + + /// Write `contents` to `relative`, recording it as produced from `source`. + /// + /// Two sources landing on one destination is an error rather than a + /// last-writer-wins overwrite: `a.by` and a hand-written `a.py` are both the + /// module `a`, and quietly picking one means the build disagrees with what + /// python will import. + pub(crate) fn write( + &mut self, + relative: &Path, + source: Option<&Path>, + contents: &str, + ) -> anyhow::Result<()> { + self.claim(relative, source)?; + let destination = self.out.join(relative); + create_parent(&destination)?; + fs::write(&destination, contents) + .with_context(|| format!("could not write {}", destination.display())) + } + + /// Copy `source` to `relative` verbatim. + pub(crate) fn copy(&mut self, relative: &Path, source: &Path) -> anyhow::Result<()> { + self.claim(relative, Some(source))?; + let destination = self.out.join(relative); + create_parent(&destination)?; + fs::copy(source, &destination).with_context(|| { + format!( + "could not copy {} to {}", + source.display(), + destination.display() + ) + })?; + Ok(()) + } + + fn claim(&mut self, relative: &Path, source: Option<&Path>) -> anyhow::Result<()> { + if let Some((previous, Some(previous_source))) = self.written.get_key_value(relative) + && Some(previous_source.as_path()) != source + { + let claimant = source.map_or_else( + || "the build".to_owned(), + |source| format!("`{}`", source.display()), + ); + anyhow::bail!( + "`{}` and {claimant} both build to `{}` — \ + they are the same module, so one of them has to be renamed", + previous_source.display(), + previous.display(), + ); + } + self.written + .insert(relative.to_path_buf(), source.map(Path::to_path_buf)); + Ok(()) + } + + /// Delete what the previous build wrote and this one did not, then record + /// what this one wrote. + pub(crate) fn finish(self) -> anyhow::Result<()> { + let manifest = self.out.join(MANIFEST_FILENAME); + let previous = read_manifest(&manifest); + let current: BTreeSet<&Path> = self.written.keys().map(PathBuf::as_path).collect(); + + let mut emptied: BTreeSet = BTreeSet::new(); + for stale in &previous { + if current.contains(stale.as_path()) { + continue; + } + let path = self.out.join(stale); + // a file the user deleted from the output themselves is already in + // the state we want, so a missing file is not an error + let _ = fs::remove_file(&path); + let mut parent = path.parent(); + while let Some(directory) = parent { + if directory == self.out { + break; + } + emptied.insert(directory.to_path_buf()); + parent = directory.parent(); + } + } + // deepest first, so a directory whose only content was other now-removed + // directories is itself removed + for directory in emptied.iter().rev() { + let _ = fs::remove_dir(directory); + } + + create_parent(&manifest)?; + let mut rendered = + String::from("# written by `by build`; delete it and stale output stays\n"); + for path in current { + rendered.push_str(&portable(path)); + rendered.push('\n'); + } + fs::write(&manifest, rendered) + .with_context(|| format!("could not write {}", manifest.display())) + } +} + +fn create_parent(path: &Path) -> anyhow::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("could not create {}", parent.display()))?; + } + Ok(()) +} + +fn read_manifest(path: &Path) -> BTreeSet { + let Ok(contents) = fs::read_to_string(path) else { + return BTreeSet::new(); + }; + contents + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + // the manifest is written with `/` separators so an output tree stays + // readable on the platform that did not write it; a path built from those + // components is native either way + .map(|line| line.split('/').collect::()) + .collect() +} + +/// A relative path with `/` separators, whatever the platform. +fn portable(path: &Path) -> String { + path.components() + .filter_map(|component| component.as_os_str().to_str()) + .collect::>() + .join("/") +} + +/// Where `source`'s output goes, relative to the output root. +/// +/// The tree mirrored is the *module* tree, not the directory tree: a src-layout +/// project's `src/pkg/main.by` is the module `pkg.main`, so it has to land at +/// `pkg/main.py`. Mirroring the directory instead emits `src/pkg/main.py`, whose +/// module is `src.pkg.main` — a name nothing imports, and one `run.main` cannot +/// sensibly be set to. A file outside every module root keeps its place relative +/// to the project. +pub(crate) fn relative_destination(roots: &[PathBuf], root: &Path, source: &Path) -> PathBuf { + let relative = roots + .iter() + .find_map(|candidate| source.strip_prefix(candidate).ok()) + .or_else(|| source.strip_prefix(root).ok()) + .unwrap_or(source); + // whatever happened above, the result has to be *relative*: joined onto the + // output directory an absolute path replaces it outright, so every emitted + // file would land outside the output tree. keeping only the named components + // also drops any `..`, which would climb back out of it + relative + .components() + .filter_map(|component| match component { + std::path::Component::Normal(name) => Some(name), + _ => None, + }) + .collect() +} + +/// What a transpiled source is called in the output. +/// +/// A stub stays a stub: `.byi` transpiles to `.pyi`, not to `.py`. Emitting a +/// stub as a module would put a body-less definition where python expects the +/// implementation, and it would shadow the real module at runtime. +pub(crate) fn transpiled_destination(roots: &[PathBuf], root: &Path, source: &Path) -> PathBuf { + let extension = match source.extension().and_then(std::ffi::OsStr::to_str) { + Some("byi") => "pyi", + _ => "py", + }; + relative_destination(roots, root, source).with_extension(extension) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roots(paths: &[&str]) -> Vec { + paths.iter().map(PathBuf::from).collect() + } + + #[test] + fn a_module_root_is_stripped() { + let destination = transpiled_destination( + &roots(&["/p/src"]), + Path::new("/p"), + Path::new("/p/src/pkg/main.by"), + ); + assert_eq!(destination, PathBuf::from("pkg/main.py")); + } + + #[test] + fn the_deepest_root_wins() { + let destination = transpiled_destination( + &roots(&["/p/src/inner", "/p/src"]), + Path::new("/p"), + Path::new("/p/src/inner/pkg/main.by"), + ); + assert_eq!(destination, PathBuf::from("pkg/main.py")); + } + + #[test] + fn a_file_outside_every_root_keeps_its_place() { + let destination = relative_destination( + &roots(&["/p/src"]), + Path::new("/p"), + Path::new("/p/assets/logo.svg"), + ); + assert_eq!(destination, PathBuf::from("assets/logo.svg")); + } + + /// a root that shares no prefix with the file — which is what `canonicalize` + /// and `current_dir` disagreeing produced on windows — must not leave the + /// path absolute: joining that onto the output directory discards the output + /// directory entirely, so every emitted file lands outside it + #[test] + fn a_root_that_does_not_match_still_yields_a_relative_path() { + let destination = transpiled_destination( + &roots(&["/other/src"]), + Path::new("/other"), + Path::new("/p/pkg/main.by"), + ); + assert!( + destination.is_relative(), + "an absolute result escapes the output directory: {}", + destination.display() + ); + } + + /// a stub is not a module: transpiling `a.byi` to `a.py` would put a + /// body-less definition where python imports the implementation + #[test] + fn a_stub_transpiles_to_a_stub() { + let destination = + transpiled_destination(&roots(&["/p"]), Path::new("/p"), Path::new("/p/a.byi")); + assert_eq!(destination, PathBuf::from("a.pyi")); + } + + #[test] + fn two_sources_claiming_one_destination_is_an_error() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut staging = Staging::new(directory.path()); + staging + .write(Path::new("a.py"), Some(Path::new("/p/a.by")), "x = 1\n") + .expect("the first write succeeds"); + let error = staging + .write(Path::new("a.py"), Some(Path::new("/p/a.py")), "x = 2\n") + .expect_err("the second source collides"); + let message = error.to_string(); + assert!(message.contains("a.by"), "{message}"); + assert!(message.contains("a.py"), "{message}"); + } + + /// the same source rewriting its own destination is not a collision — that is + /// just a rebuild + #[test] + fn one_source_may_claim_its_destination_twice() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut staging = Staging::new(directory.path()); + let source = PathBuf::from("/p/a.by"); + staging + .write(Path::new("a.py"), Some(&source), "x = 1\n") + .expect("the first write succeeds"); + staging + .write(Path::new("a.py"), Some(&source), "x = 2\n") + .expect("rewriting from the same source is fine"); + } + + #[test] + fn what_the_previous_build_wrote_and_this_one_did_not_is_deleted() { + let directory = tempfile::tempdir().expect("tempdir"); + + let mut first = Staging::new(directory.path()); + first + .write(Path::new("kept.py"), None, "x = 1\n") + .expect("write"); + first + .write(Path::new("pkg/gone.py"), None, "x = 1\n") + .expect("write"); + first.finish().expect("finish"); + assert!(directory.path().join("pkg/gone.py").exists()); + + let mut second = Staging::new(directory.path()); + second + .write(Path::new("kept.py"), None, "x = 1\n") + .expect("write"); + second.finish().expect("finish"); + + assert!(directory.path().join("kept.py").exists()); + assert!( + !directory.path().join("pkg/gone.py").exists(), + "a module the source no longer has must not survive in the output" + ); + assert!( + !directory.path().join("pkg").exists(), + "the directory it was the only content of goes with it" + ); + } + + /// only what the build itself wrote is ever deleted. anything else in the + /// output directory was put there by someone, and a build is not entitled to + /// remove it + #[test] + fn a_file_the_build_never_wrote_is_left_alone() { + let directory = tempfile::tempdir().expect("tempdir"); + fs::write(directory.path().join("theirs.txt"), "hands off").expect("write"); + + let mut staging = Staging::new(directory.path()); + staging + .write(Path::new("mine.py"), None, "x = 1\n") + .expect("write"); + staging.finish().expect("finish"); + + Staging::new(directory.path()).finish().expect("finish"); + + assert!(directory.path().join("theirs.txt").exists()); + assert!(!directory.path().join("mine.py").exists()); + } + + #[test] + fn a_manifest_round_trips_through_its_portable_form() { + let directory = tempfile::tempdir().expect("tempdir"); + let mut staging = Staging::new(directory.path()); + staging + .write(&PathBuf::from("pkg").join("deep").join("a.py"), None, "") + .expect("write"); + staging.finish().expect("finish"); + + let manifest = read_manifest(&directory.path().join(MANIFEST_FILENAME)); + assert!(manifest.contains(&PathBuf::from("pkg").join("deep").join("a.py"))); + let rendered = fs::read_to_string(directory.path().join(MANIFEST_FILENAME)).expect("read"); + assert!( + rendered.contains("pkg/deep/a.py"), + "the manifest is written with `/` separators:\n{rendered}" + ); + } +} diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index 783be90994..d34daf8a74 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -1,6 +1,8 @@ mod args; mod by_commands; +mod by_init; mod by_source_encoding; +mod by_staging; mod logging; mod printer; mod python_version; @@ -107,6 +109,7 @@ fn run_command(command: Command) -> anyhow::Result { module, args, min_version, + python, lowering, compiled, } => by_commands::cmd_run( @@ -115,11 +118,30 @@ fn run_command(command: Command) -> anyhow::Result { min_version.as_deref(), &lowering, compiled, + python.as_deref(), ), + Command::Init { + path, + name, + lib, + app: _, + python_version, + } => { + let kind = if lib { + by_init::ProjectKind::Library + } else { + by_init::ProjectKind::Application + }; + let version = python_version + .unwrap_or_else(|| by_commands::default_project_python_version().to_string()); + by_init::cmd_init(path.as_deref(), name.as_deref(), kind, &version) + } Command::Build { min_version, + out, + print_manifest, lowering, - } => by_commands::cmd_build(min_version.as_deref(), &lowering), + } => by_commands::cmd_build(min_version.as_deref(), &lowering, &out, print_manifest), Command::Compile { files, output, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 84d61c62f6..b305660942 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -2795,3 +2795,710 @@ def main(): "expected the override to be reported once enabled:\n{rendered}" ); } + +// ── building a project, not just its `.by` files ───────────────────────────── + +/// a project is its hand-written python too. an output tree holding only the +/// transpiled half is not a project: the first `import` of a `.py` sibling +/// fails, and there is nothing the author can do about it from the `.by` side +#[test] +fn build_carries_a_python_module_into_the_output() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "from helper import shout\n").unwrap(); + fs::write( + dir.path().join("helper.py"), + "def shout(text: str) -> str:\n return text.upper()\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert_eq!( + fs::read_to_string(dir.path().join("out/helper.py")).unwrap(), + "def shout(text: str) -> str:\n return text.upper()\n", + "a hand-written python module belongs in the output verbatim" + ); +} + +/// and its data. a program that opens a file beside itself is the ordinary case, +/// not an exotic one +#[test] +fn build_carries_data_files_into_the_output() { + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write(package.join("settings.json"), "{\"key\": 1}\n").unwrap(); + fs::write(package.join("py.typed"), "").unwrap(); + fs::write(package.join("template.html"), "

hi

\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + let out = dir.path().join("out").join("app"); + assert_eq!( + fs::read_to_string(out.join("settings.json")).unwrap(), + "{\"key\": 1}\n" + ); + assert!(out.join("py.typed").exists()); + assert!(out.join("template.html").exists()); +} + +/// a stub is not a module: emitting `a.byi` as `a.py` would put a body-less +/// definition where python imports the implementation, and shadow the real +/// module at runtime +#[test] +fn build_writes_a_stub_as_a_stub() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + fs::write(dir.path().join("shapes.byi"), "def area() -> int: ...\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!( + dir.path().join("out/shapes.pyi").exists(), + "a `.byi` builds to a `.pyi`:\n{stderr}" + ); + assert!( + !dir.path().join("out/shapes.py").exists(), + "a stub emitted as a module shadows the implementation" + ); +} + +/// `a.by` and a hand-written `a.py` are both the module `a`. picking one and +/// carrying on means the build disagrees with what python will import, so this +/// is reported rather than resolved +#[test] +fn build_refuses_two_sources_that_are_one_module() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("thing.by"), "x = 1\n").unwrap(); + fs::write(dir.path().join("thing.py"), "x = 2\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "a collision must fail the build:\n{stderr}" + ); + assert!( + stderr.contains("same module"), + "the collision must say what is wrong:\n{stderr}" + ); +} + +/// an output tree that only ever grows keeps a module that was deleted months +/// ago importable — locally, where nobody notices, and then in the wheel built +/// from the same tree, where somebody does +#[test] +fn build_deletes_output_the_project_no_longer_has() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("kept.by"), "x = 1\n").unwrap(); + fs::write(dir.path().join("removed.by"), "y = 2\n").unwrap(); + + let build = || { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + + build(); + assert!(dir.path().join("out/removed.py").exists()); + + fs::remove_file(dir.path().join("removed.by")).unwrap(); + build(); + + assert!(dir.path().join("out/kept.py").exists()); + assert!( + !dir.path().join("out/removed.py").exists(), + "output for a source that is gone must not survive the next build" + ); +} + +/// only what the build itself wrote is ever deleted — anything else in the +/// output directory was put there by somebody +#[test] +fn build_leaves_output_it_never_wrote_alone() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + fs::create_dir_all(dir.path().join("out")).unwrap(); + fs::write(dir.path().join("out/theirs.txt"), "hands off\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(dir.path().join("out/theirs.txt").exists()); +} + +#[test] +fn build_writes_where_out_says() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--out", "elsewhere"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!(dir.path().join("elsewhere/main.py").exists()); + assert!(!dir.path().join("out").exists()); +} + +/// the output directory is not an input to itself, wherever it is put +#[test] +fn build_does_not_read_its_own_output() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + + for _ in 0..2 { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--out", "elsewhere"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + assert!( + !dir.path().join("elsewhere/elsewhere").exists(), + "a second build must not copy the first build's output into itself" + ); +} + +/// a source distribution has to carry exactly what the build read, and a wheel +/// exactly the packages it produced. both are the build's answers +#[test] +fn build_reports_what_it_read_and_what_it_produced() { + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("src").join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(dir.path().join("README.md"), "# app\n").unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write(package.join("helper.py"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--print-manifest"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let listed: Vec<&str> = stdout.lines().collect(); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + for expected in ["README.md", "src/app/__init__.by", "src/app/helper.py"] { + let expected = format!( + "input {}", + expected.replace('/', std::path::MAIN_SEPARATOR_STR) + ); + assert!( + listed.contains(&expected.as_str()), + "`{expected}` is part of this project:\n{stdout}" + ); + } + assert!( + listed.contains(&"package app"), + "the package the wheel ships:\n{stdout}" + ); + assert_eq!( + listed + .iter() + .filter(|line| line.ends_with("__init__.by")) + .count(), + 1, + "a source that produced two outputs is still one input:\n{stdout}" + ); +} + +/// `tests` beside `src` is a package python can import and a package nobody +/// installs. a wheel that shipped it would put a top-level `tests` module into +/// every environment the project is installed into +#[test] +fn build_does_not_ship_what_lives_outside_the_source_root() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"app\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let package = dir.path().join("src").join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + let tests = dir.path().join("tests"); + fs::create_dir_all(&tests).unwrap(); + fs::write(tests.join("__init__.py"), "").unwrap(); + fs::write(tests.join("test_it.py"), "def test_x(): pass\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--print-manifest"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let listed: Vec<&str> = stdout.lines().collect(); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(listed.contains(&"package app"), "{stdout}"); + assert!( + !listed.contains(&"package tests"), + "`tests` is not part of the distribution:\n{stdout}" + ); + // it is still built, because it is still the project — running the tests out + // of the output tree is the point of building them + assert!(dir.path().join("out/tests/test_it.py").exists()); + assert!( + !dir.path().join("out/tests/by.typed").exists(), + "a marker only speaks for what the project ships" + ); +} + +/// the marker is what tells a downstream basedpython project to read the `.by` +/// beside a module rather than the python it was transpiled into +#[test] +fn build_marks_a_package_as_carrying_its_sources() { + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write(package.join("deep.by"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + let out = dir.path().join("out").join("app"); + assert!( + out.join("by.typed").exists(), + "expected a marker:\n{stderr}" + ); + assert!( + out.join("deep.by").exists(), + "the marker is a claim about sources, which have to be there:\n{stderr}" + ); + assert!(out.join("deep.py").exists()); +} + +#[test] +fn build_ships_python_only_when_the_project_says_so() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.build]\nsources = false\n", + ) + .unwrap(); + let package = dir.path().join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + let out = dir.path().join("out").join("app"); + assert!(out.join("__init__.py").exists()); + assert!( + !out.join("__init__.by").exists(), + "`sources = false` ships python only" + ); + // the marker still goes out. its precedence claim is vacuous without sources + // — there is no `.by` to prefer — but its contents are what declare which + // dependencies this project hands out on purpose, and a python-only build has + // those too + assert!( + out.join("by.typed").exists(), + "the marker carries the export declaration, sources or no sources" + ); +} + +#[test] +fn build_honours_the_configured_exclusions() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.build]\nexclude = [\"secrets.json\"]\n", + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + fs::write(dir.path().join("secrets.json"), "{}\n").unwrap(); + fs::write(dir.path().join("public.json"), "{}\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!(dir.path().join("out/public.json").exists()); + assert!( + !dir.path().join("out/secrets.json").exists(), + "an excluded file must not reach the output" + ); +} + +/// a directory ty's defaults drop can be taken back with a negated exclude, and +/// the build has to honour that for every file in it — not just the `.by` ones. +/// re-dropping the rest would leave the transpiled half of a directory the +/// project deliberately re-included +#[test] +fn build_carries_a_directory_a_negated_exclude_takes_back() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.src]\nexclude = [\"!dist\"]\n", + ) + .unwrap(); + let generated = dir.path().join("dist"); + fs::create_dir_all(&generated).unwrap(); + fs::write(generated.join("kept.by"), "x = 1\n").unwrap(); + fs::write(generated.join("kept.json"), "{}\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!( + dir.path().join("out/dist/kept.py").exists(), + "the re-included `.by` builds:\n{stderr}" + ); + assert!( + dir.path().join("out/dist/kept.json").exists(), + "and so does everything beside it:\n{stderr}" + ); +} + +/// the rule follows the module tree rather than the name `src`. a `src` that is +/// itself a package is not a source root, so the module really is `src.mymod` — +/// and a wheel that dropped the `src` component would ship a package under a name +/// nothing imports +#[test] +fn build_ships_a_source_directory_that_is_itself_a_package() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"mymod\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let package = dir.path().join("src").join("mymod"); + fs::create_dir_all(&package).unwrap(); + fs::write(dir.path().join("src").join("__init__.py"), "").unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--print-manifest"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + stdout.lines().any(|line| line == "package src"), + "`src.mymod` is the module, so `src` is the package:\n{stdout}" + ); + assert!(dir.path().join("out/src/mymod/__init__.py").exists()); +} + +// ── running a project, not just its `.by` files ────────────────────────────── + +/// the same hole at run time, where it is fatal rather than untidy: `by run` +/// executes out of a directory it stages, so a `.py` module missing from it +/// cannot be imported at all +#[test] +fn run_imports_a_python_module_beside_the_transpiled_ones() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "from helper import shout\n\nprint(shout(\"mixed\"))\n", + ) + .unwrap(); + fs::write( + dir.path().join("helper.py"), + "def shout(text: str) -> str:\n return text.upper()\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "MIXED"); +} + +#[test] +fn run_reads_a_data_file_beside_the_program() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "from pathlib import Path\n\n\ + print(Path(__file__).parent.joinpath(\"greeting.txt\").read_text().strip())\n", + ) + .unwrap(); + fs::write(dir.path().join("greeting.txt"), "read from disk\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!( + output.status.success(), + "by run failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "read from disk" + ); +} + +/// running a project on an interpreter older than it targets used to fail as a +/// `SyntaxError` inside generated code, in a temporary directory that was +/// already deleted. it is knowable before anything runs, so it is said before +/// anything runs +#[test] +fn run_refuses_an_interpreter_older_than_the_project_targets() { + let dir = tempfile::tempdir().expect("tempdir"); + let (major, minor) = running_python_version(); + let unreachable = format!("{major}.{}", minor + 1); + fs::write( + dir.path().join("pyproject.toml"), + format!( + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + requires-python = \">={unreachable}\"\n" + ), + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + !output.status.success(), + "a project that cannot run on this interpreter must say so:\n{stderr}" + ); + assert!( + stderr.contains(&format!("targets python {unreachable}")), + "the message has to name both versions:\n{stderr}" + ); + assert!( + stderr.contains("--min-version"), + "and what to do about it:\n{stderr}" + ); +} + +/// the version of the interpreter `by run` would pick, so a test can name one +/// that is definitely newer +fn running_python_version() -> (u8, u8) { + let output = Command::new("python3") + .args([ + "-c", + "import sys; print(f'{sys.version_info[0]} {sys.version_info[1]}')", + ]) + .output() + .expect("python3 is needed to run this test"); + let rendered = String::from_utf8_lossy(&output.stdout); + let mut parts = rendered.split_whitespace(); + let major = parts.next().unwrap().parse().unwrap(); + let minor = parts.next().unwrap().parse().unwrap(); + (major, minor) +} + +/// the shim `by run` puts in the tree it executes is written through the same +/// staging as everything else, so a project file of that name is a reported +/// collision rather than a silent overwrite +#[test] +fn run_refuses_a_project_file_that_collides_with_its_shim() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + fs::write(dir.path().join("_by_runner.py"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "expected a refusal:\n{stderr}"); + assert!(stderr.contains("_by_runner.py"), "{stderr}"); + assert!(stderr.contains("same module"), "{stderr}"); +} + +/// a compiler's output directory is not project source, and it is the one most +/// likely to be enormous — this used to be copied in full on every build and +/// every run +#[test] +fn build_does_not_carry_a_compilers_output_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + let artifacts = dir.path().join("target").join("debug"); + fs::create_dir_all(&artifacts).unwrap(); + fs::write(artifacts.join("blob"), "an enormous binary").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!(dir.path().join("out").join("main.py").exists()); + assert!( + !dir.path().join("out").join("target").exists(), + "a build directory must not be carried into the build:\n{stderr}" + ); +} + +// ── starting a project ─────────────────────────────────────────────────────── + +/// what `by init` writes has to be a project the rest of the toolchain accepts, +/// or it is a template for a thing that does not work +#[test] +fn init_writes_a_project_that_builds_and_runs() { + let dir = tempfile::tempdir().expect("tempdir"); + let (major, minor) = running_python_version(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args([ + "init", + "demo", + "--python-version", + &format!("{major}.{minor}"), + ]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by init failed:\n{stderr}"); + + let project = dir.path().join("demo"); + let pyproject = fs::read_to_string(project.join("pyproject.toml")).unwrap(); + assert!(pyproject.contains("build-backend = \"basedpython.build\"")); + assert!(project.join("src/demo/__init__.by").exists()); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("run") + .current_dir(&project) + .output() + .expect("failed to spawn by"); + assert!( + output.status.success(), + "a new project has to run:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "hello from basedpython" + ); +} + +#[test] +fn init_refuses_to_write_over_a_project() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"already-here\"\nversion = \"9.9.9\"\n", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("init") + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "expected a refusal:\n{stderr}"); + assert!(stderr.contains("already exists"), "{stderr}"); + assert!( + fs::read_to_string(dir.path().join("pyproject.toml")) + .unwrap() + .contains("9.9.9"), + "the existing project must be untouched" + ); +} diff --git a/crates/ty/tests/cli/config_option.rs b/crates/ty/tests/cli/config_option.rs index 7f342c38cf..fd6fe53d04 100644 --- a/crates/ty/tests/cli/config_option.rs +++ b/crates/ty/tests/cli/config_option.rs @@ -123,7 +123,7 @@ fn cli_config_args_invalid_option() -> anyhow::Result<()> { | 1 | bad-option=true | ^^^^^^^^^^ - unknown field `bad-option`, expected one of `environment`, `src`, `rules`, `terminal`, `analysis`, `run`, `editor`, `overrides` + unknown field `bad-option`, expected one of `environment`, `src`, `rules`, `terminal`, `analysis`, `run`, `build`, `editor`, `overrides` Usage: by diff --git a/crates/ty_module_resolver/src/list.rs b/crates/ty_module_resolver/src/list.rs index ba421da8ae..01edbccdce 100644 --- a/crates/ty_module_resolver/src/list.rs +++ b/crates/ty_module_resolver/src/list.rs @@ -273,7 +273,12 @@ impl<'db> Lister<'db> { return; } - let Some(file) = module_path.to_file(&self.context()) else { + // which file *is* this module is resolution's decision, not the walk's: a + // `.pyi` outranks a `.py`, and a `.by` outranks a `.py` in a package that + // declares its sources authoritative. taking whichever file the directory + // happened to yield first would mean a name meant one thing when it was + // offered and another when it was used + let Some(file) = resolve_file_module(&module_path, &self.context()) else { return; }; self.add_module( diff --git a/crates/ty_module_resolver/src/path.rs b/crates/ty_module_resolver/src/path.rs index 54829ac86d..ee18a3dc35 100644 --- a/crates/ty_module_resolver/src/path.rs +++ b/crates/ty_module_resolver/src/path.rs @@ -176,6 +176,36 @@ impl ModulePath { } } + /// Whether the directory this module *lives in* declares its `.by` sources + /// to be the authoritative surface. + /// + /// A basedpython library ships both halves: the transpiled `.py` python + /// imports, and the `.by` it was transpiled from. Only the `.by` still says + /// the things python cannot spell — `extension` blocks, `raises` clauses, + /// read-only `let`, sum types — so for type checking it is the better of the + /// two, and a `by.typed` marker beside them is how the package says so. + /// Without the marker the python wins, exactly as it did before any of this. + pub(super) fn by_typed(&self, resolver: &ResolverContext) -> bool { + let Some(path) = self.to_system_path() else { + return false; + }; + let Some(directory) = path.parent() else { + return false; + }; + directory_declares_by_typed(resolver, directory) + } + + /// Whether this package directory itself carries the marker. + /// + /// A package declares it once, at the top: the resolution walk carries it + /// down to every module underneath, the same way `py.typed` is inherited. + pub(super) fn declares_by_typed(&self, resolver: &ResolverContext) -> bool { + let Some(path) = self.to_system_path() else { + return false; + }; + directory_declares_by_typed(resolver, &path) + } + /// Get the `py.typed` info for this package (not considering parent packages) pub(super) fn py_typed(&self, resolver: &ResolverContext) -> PyTyped { let Some(py_typed_contents) = self.to_system_path().and_then(|path| { @@ -410,6 +440,21 @@ impl PartialEq for VendoredPathBuf { } } +/// Whether `directory` is a package whose `.by` sources are authoritative. +/// +/// The marker only counts inside a package. A stray `by.typed` loose in +/// `site-packages` would otherwise re-point every top-level module in the +/// environment, and no package has any business making that claim for its +/// neighbours. +fn directory_declares_by_typed(resolver: &ResolverContext, directory: &SystemPath) -> bool { + directory_contains_file(resolver.db, directory, &[crate::BY_TYPED]) + && directory_contains_file( + resolver.db, + directory, + &["__init__.py", "__init__.pyi", "__init__.by", "__init__.byi"], + ) +} + fn directory_contains_file(db: &dyn Db, directory: &SystemPath, names: &[&str]) -> bool { let Ok(listing) = directory_listing(db, directory) else { return false; diff --git a/crates/ty_module_resolver/src/resolve.rs b/crates/ty_module_resolver/src/resolve.rs index 60ccf9ef57..a859db5a0a 100644 --- a/crates/ty_module_resolver/src/resolve.rs +++ b/crates/ty_module_resolver/src/resolve.rs @@ -1187,6 +1187,8 @@ struct ModuleResolutionCandidate { path: ModulePath, module: ResolvedModule, py_typed: PyTyped, + /// whether an enclosing package declared its `.by` sources authoritative + by_typed: bool, precedence: CandidatePrecedence, } @@ -1204,6 +1206,7 @@ impl ModuleResolutionCandidate { path: search_path.to_module_path(), module: ResolvedModule::NamespacePackage, py_typed: PyTyped::Untyped, + by_typed: false, precedence, } } @@ -1612,17 +1615,21 @@ fn resolve_component( return Err(()); } + let by_typed = candidate.by_typed; let package_path = &mut candidate.path; package_path.push(module_name); // Check for a regular package first (highest priority) package_path.push("__init__"); - if let Some(init) = resolve_file_module_with_filter(package_path, context, file_filter) { + if let Some(init) = + resolve_file_module_with_filter(package_path, context, file_filter, by_typed) + { // Remove the `__init__` component for any potential next step package_path.pop(); candidate.py_typed = package_path .py_typed(context) .inherit_parent(candidate.py_typed); + candidate.by_typed = by_typed || package_path.declares_by_typed(context); if is_legacy_namespace_package(package_path, context, init) { candidate.module = ResolvedModule::LegacyNamespacePackage(init); } else { @@ -1634,7 +1641,9 @@ fn resolve_component( // Check for a file module next package_path.pop(); - if let Some(file_module) = resolve_file_module_with_filter(package_path, context, file_filter) { + if let Some(file_module) = + resolve_file_module_with_filter(package_path, context, file_filter, by_typed) + { candidate.module = ResolvedModule::Module(file_module); return Ok(()); } @@ -1701,13 +1710,19 @@ pub(super) fn resolve_file_module( module: &ModulePath, resolver_state: &ResolverContext, ) -> Option { - resolve_file_module_with_filter(module, resolver_state, ComponentFileFilter::ByMode) + resolve_file_module_with_filter(module, resolver_state, ComponentFileFilter::ByMode, false) } +/// Resolve `module` to a file. +/// +/// `by_typed` is whether an enclosing package already declared its `.by` sources +/// authoritative; a marker on the module's own directory counts for just as much, +/// and is only looked for when there is a `.py` for it to outrank. fn resolve_file_module_with_filter( module: &ModulePath, resolver_state: &ResolverContext, filter: ComponentFileFilter, + by_typed: bool, ) -> Option { let stub_file = if resolver_state.mode.is_typing() { module.with_pyi_extension().to_file(resolver_state) @@ -1724,18 +1739,30 @@ fn resolve_file_module_with_filter( return stub_file.or(by_stub_file); } - stub_file - .or(by_stub_file) - .or_else(|| { - module - .with_py_extension() - .and_then(|path| path.to_file(resolver_state)) - }) - .or_else(|| { - module - .with_by_extension() - .and_then(|path| path.to_file(resolver_state)) - }) + let by_source = || { + module + .with_by_extension() + .and_then(|path| path.to_file(resolver_state)) + }; + let py_source = module + .with_py_extension() + .and_then(|path| path.to_file(resolver_state)); + + // a `.by` beside a `.py` is the source the `.py` was generated from, whether + // that happened in a wheel or in the project itself. which of the two answers + // is the module is the package's to declare + let source = match py_source { + Some(py_source) => { + if by_typed || module.by_typed(resolver_state) { + by_source().or(Some(py_source)) + } else { + Some(py_source) + } + } + None => by_source(), + }; + + stub_file.or(by_stub_file).or(source) } /// Determines whether a package is a legacy namespace package. diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index 76f9b3fe59..5571505e26 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -5,7 +5,7 @@ use crate::glob::{ PortableGlobKind, }; use crate::metadata::python_version::SupportedPythonVersion; -use crate::metadata::settings::{OverrideSettings, SrcSettings}; +use crate::metadata::settings::{BuildSettings, OverrideSettings, SrcSettings}; use super::settings::{EditorSettings, Override, Settings, TerminalSettings}; use crate::metadata::value::{RelativeGlobPattern, RelativePathBuf}; @@ -110,6 +110,11 @@ pub struct Options { #[option_group] pub run: Option, + /// Configures what `by build` writes, and what a wheel of this project carries. + #[serde(skip_serializing_if = "Option::is_none")] + #[option_group] + pub build: Option, + /// Configures the parts of the editor experience that type checking does not decide. #[serde(skip_serializing_if = "Option::is_none")] #[option_group] @@ -476,6 +481,17 @@ impl Options { }); let src = strategy.fallback(src, |_| SrcSettings::default())?; + let build = self + .build + .or_default() + .to_settings(db, project_root, &mut diagnostics) + .map_err(|err| ToSettingsError { + diagnostic: err, + output_format: terminal.output_format, + color: colored::control::SHOULD_COLORIZE.should_colorize(), + }); + let build = strategy.fallback(build, |_| BuildSettings::default())?; + let mut analysis_diagnostics = Vec::new(); let analysis = self .analysis @@ -515,6 +531,7 @@ impl Options { rules: Arc::new(rules), terminal, src, + build, analysis, editor, overrides, @@ -1417,6 +1434,8 @@ enum GlobFilterContext { SrcRoot, /// Override configuration context Overrides, + /// Build output configuration context + Build, } impl GlobFilterContext { @@ -1424,6 +1443,7 @@ impl GlobFilterContext { match self { Self::SrcRoot => "src.include", Self::Overrides => "overrides.include", + Self::Build => "build.include", } } @@ -1431,6 +1451,7 @@ impl GlobFilterContext { match self { Self::SrcRoot => "src.exclude", Self::Overrides => "overrides.exclude", + Self::Build => "build.exclude", } } } @@ -1583,6 +1604,137 @@ pub struct RunOptions { pub main: Option>, } +#[derive( + Debug, + Default, + Clone, + Eq, + PartialEq, + Hash, + Combine, + Serialize, + Deserialize, + OptionsMetadata, + get_size2::GetSize, +)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct BuildOptions { + /// Files to carry into the build output verbatim, in addition to the ones + /// that are there by default. + /// + /// `by build` mirrors the whole module tree: a `.by` file is transpiled, and + /// every other file — a hand-written `.py`, a `py.typed` marker, a template, + /// a data file — is copied to the same place in the output. `include` is for + /// the files that sit *outside* a module root and still belong in the build, + /// such as a data directory next to `src`. + /// + /// The syntax is the same as `src.include`, and paths are anchored to the + /// project root. `exclude` takes precedence over `include`. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"null"#, + value_type = r#"list[str]"#, + example = r#" + include = [ + "assets", + ] + "# + )] + pub include: Option>>, + + /// Files to keep out of the build output. + /// + /// The syntax is the same as `src.exclude`, and paths are anchored to the + /// project root. Excluding a `.by` file keeps its transpiled output out of + /// the build as well. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"null"#, + value_type = r#"list[str]"#, + example = r#" + exclude = [ + "tests", + "**/*.snapshot", + ] + "# + )] + pub exclude: Option>>, + + /// Whether the build output carries the `.by` sources alongside the python + /// they were transpiled into, with a `by.typed` marker naming them as the + /// authoritative surface. + /// + /// This is what lets one basedpython project depend on another: a downstream + /// python project reads the transpiled `.py` and is served perfectly, while a + /// downstream basedpython project reads the `.by` and keeps the declarations + /// that have no python spelling — `extension` blocks, `raises` clauses, + /// read-only `let`, sum types. + /// + /// Enabled by default. Turn it off to ship python only. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"true"#, + value_type = "bool", + example = r#" + sources = false + "# + )] + pub sources: Option, + + /// The module to read `__version__` from, when `[project]` declares + /// `dynamic = ["version"]`. + /// + /// This is read when a wheel or a source distribution is built, not by the + /// checker: a version has to be settled before the packaging backend sees the + /// project, and the place it lives is a `.by` module that backend cannot + /// read. + /// + /// The value is a path relative to the project root. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"null"#, + value_type = "str", + example = r#" + version-from = "src/app/__init__.by" + "# + )] + pub version_from: Option>, +} + +impl BuildOptions { + fn to_settings( + &self, + db: &dyn Db, + project_root: &SystemPath, + diagnostics: &mut Vec, + ) -> Result> { + let include = build_include_filter( + db, + project_root, + self.include.as_ref(), + GlobFilterContext::Build, + diagnostics, + )?; + // no default patterns of its own: the build is already bounded by + // `src.exclude`, defaults and all, and applying them a second time here + // would re-drop whatever a negation there deliberately took back + let exclude = build_exclude_filter( + db, + project_root, + self.exclude.as_ref(), + &[], + GlobFilterContext::Build, + diagnostics, + )?; + + Ok(BuildSettings { + files: IncludeExcludeFilter::new(include, exclude), + sources: self.sources.unwrap_or(true), + }) + } +} + #[derive( Debug, Default, diff --git a/crates/ty_project/src/metadata/settings.rs b/crates/ty_project/src/metadata/settings.rs index 00000ed61d..4ccae92166 100644 --- a/crates/ty_project/src/metadata/settings.rs +++ b/crates/ty_project/src/metadata/settings.rs @@ -7,6 +7,9 @@ use ty_python_semantic::lint::RuleSelection; use crate::metadata::options::{FileOptions, InnerOverrideOptions, Options, OutputFormat}; use crate::metadata::script::script_metadata; +use ruff_db::system::SystemPath; + +use crate::glob::{GlobFilterCheckMode, IncludeResult}; use crate::{Db, glob::IncludeExcludeFilter}; /// The resolved [`super::Options`] for the project. @@ -27,6 +30,7 @@ pub struct Settings { pub(super) rules: Arc, pub(super) terminal: TerminalSettings, pub(super) src: SrcSettings, + pub(super) build: BuildSettings, pub(super) analysis: AnalysisSettings, pub(super) editor: EditorSettings, @@ -47,6 +51,10 @@ impl Settings { &self.src } + pub fn build(&self) -> &BuildSettings { + &self.build + } + pub(crate) fn to_rules(&self) -> Arc { self.rules.clone() } @@ -127,6 +135,28 @@ pub struct SrcSettings { pub(crate) files: IncludeExcludeFilter, } impl SrcSettings { + /// Whether this file is part of the project's own source. + /// + /// The file set the checker walks is derived from this; a build asks it too, + /// so that a file the project excludes from itself does not turn up in what + /// the project ships. + pub fn is_file_included(&self, path: &SystemPath) -> bool { + matches!( + self.files + .is_file_included(path, GlobFilterCheckMode::Adhoc), + IncludeResult::Included { .. } + ) + } + + /// Whether a walk should descend into this directory. + pub fn is_directory_included(&self, path: &SystemPath) -> bool { + !matches!( + self.files + .is_directory_maybe_included(path, GlobFilterCheckMode::Adhoc), + IncludeResult::Excluded + ) + } + pub(crate) fn default() -> Self { Self { respect_ignore_files: true, @@ -136,6 +166,51 @@ impl SrcSettings { } } +/// The resolved `[tool.basedpython.build]` options. +#[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] +pub struct BuildSettings { + pub(super) files: IncludeExcludeFilter, + pub(super) sources: bool, +} + +impl BuildSettings { + /// Whether this file belongs in the build output. + /// + /// The filter is the only thing consulted here: the build walks the module + /// tree itself, so "is this file part of the project at all" has already been + /// decided by the time a path reaches this. + pub fn is_file_included(&self, path: &SystemPath) -> bool { + matches!( + self.files + .is_file_included(path, GlobFilterCheckMode::Adhoc), + IncludeResult::Included { .. } + ) + } + + /// Whether the build walk should descend into this directory. + pub fn is_directory_included(&self, path: &SystemPath) -> bool { + !matches!( + self.files + .is_directory_maybe_included(path, GlobFilterCheckMode::Adhoc), + IncludeResult::Excluded + ) + } + + /// Whether the output carries the `.by` sources alongside their transpiled python. + pub fn sources(&self) -> bool { + self.sources + } +} + +impl Default for BuildSettings { + fn default() -> Self { + Self { + files: IncludeExcludeFilter::default(), + sources: true, + } + } +} + /// A single configuration override that applies to files matching specific patterns. #[derive(Debug, Clone, PartialEq, Eq, get_size2::GetSize)] pub struct Override { diff --git a/crates/ty_python_semantic/resources/mdtest/import/by_typed_packages.md b/crates/ty_python_semantic/resources/mdtest/import/by_typed_packages.md new file mode 100644 index 0000000000..a0e939ea93 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/import/by_typed_packages.md @@ -0,0 +1,229 @@ +# Packages that carry their basedpython sources + +A basedpython library ships both halves of itself: the transpiled `.py` that python imports, and the +`.by` those were transpiled from. Only the `.by` still says the things python has no spelling for, +so for type checking it is the better of the two — but which one is authoritative is the package's +claim to make, not the checker's guess. A `by.typed` marker beside them is how a package makes it, +in the same spirit as `py.typed`. + +Without the marker, python wins. + +## Without a marker, the python is the module + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/shipped/__init__.py`: + +```py +``` + +`/packages/shipped/value.py`: + +```py +NUMBER: int = 1 +``` + +`/packages/shipped/value.by`: + +```by +NUMBER: str = "from the source" +``` + +```py +from shipped.value import NUMBER + +reveal_type(NUMBER) # revealed: int +``` + +## With a marker, the basedpython source is + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/shipped/by.typed`: + +```text +``` + +`/packages/shipped/__init__.py`: + +```py +``` + +`/packages/shipped/value.py`: + +```py +NUMBER: int = 1 +``` + +`/packages/shipped/value.by`: + +```by +NUMBER: str = "from the source" +``` + +```py +from shipped.value import NUMBER + +reveal_type(NUMBER) # revealed: str +``` + +## The package's own `__init__` is covered by its marker + +The marker sits beside `__init__`, so it has to be read before `__init__` itself resolves — +otherwise the one module every import of the package touches is the one module the marker does not +reach. + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/shipped/by.typed`: + +```text +``` + +`/packages/shipped/__init__.py`: + +```py +NAME: int = 1 +``` + +`/packages/shipped/__init__.by`: + +```by +NAME: str = "from the source" +``` + +```py +from shipped import NAME + +reveal_type(NAME) # revealed: str +``` + +## A marker is inherited by subpackages + +A package declares it once, at the top. Every module underneath is part of that same distribution +and was shipped by the same build. + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/shipped/by.typed`: + +```text +``` + +`/packages/shipped/__init__.py`: + +```py +``` + +`/packages/shipped/inner/__init__.py`: + +```py +``` + +`/packages/shipped/inner/deep.py`: + +```py +NUMBER: int = 1 +``` + +`/packages/shipped/inner/deep.by`: + +```by +NUMBER: str = "from the source" +``` + +```py +from shipped.inner.deep import NUMBER + +reveal_type(NUMBER) # revealed: str +``` + +## A stub still outranks both + +The marker settles which *source* is authoritative. It says nothing about stubs, which outrank +source either way. + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/shipped/by.typed`: + +```text +``` + +`/packages/shipped/__init__.py`: + +```py +``` + +`/packages/shipped/value.pyi`: + +```pyi +NUMBER: bytes +``` + +`/packages/shipped/value.py`: + +```py +NUMBER: int = 1 +``` + +`/packages/shipped/value.by`: + +```by +NUMBER: str = "from the source" +``` + +```py +from shipped.value import NUMBER + +reveal_type(NUMBER) # revealed: bytes +``` + +## A marker loose outside a package claims nothing + +A marker only speaks for the package it is in. Left directly in a search path it would otherwise +re-point every top-level module in the environment, which is not a claim any one package has any +business making for its neighbours. + +```toml +[environment] +extra-paths = ["/packages"] +``` + +`/packages/by.typed`: + +```text +``` + +`/packages/loose.py`: + +```py +NUMBER: int = 1 +``` + +`/packages/loose.by`: + +```by +NUMBER: str = "from the source" +``` + +```py +from loose import NUMBER + +reveal_type(NUMBER) # revealed: int +``` diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index 0653c983d4..c2c8826b3f 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -180,6 +180,23 @@ Settings: Settings { }, }, }, + build: BuildSettings { + files: IncludeExcludeFilter { + include: IncludeFilter( + [ + "**", + ], + .. + ), + exclude: ExcludeFilter { + ignore: Gitignore( + [], + .. + ), + }, + }, + sources: true, + }, analysis: AnalysisSettings { strict_generic_narrowing: false, strict_equality_semantics: false, diff --git a/crates/ty_site_packages/src/lib.rs b/crates/ty_site_packages/src/lib.rs index 7649164acf..7ff9a793d7 100644 --- a/crates/ty_site_packages/src/lib.rs +++ b/crates/ty_site_packages/src/lib.rs @@ -404,6 +404,41 @@ impl PythonEnvironment { pub fn is_virtual(&self) -> bool { matches!(self, Self::Virtual(_)) } + + /// This environment's `sys.prefix`. + pub fn sys_prefix(&self) -> &SystemPath { + match self { + Self::Virtual(env) => &env.root_path, + Self::System(env) => env.path.sys_prefix(), + } + } + + /// The interpreter this environment runs, if it has one. + /// + /// Type checking only ever needs the search paths, but `by run` needs the + /// executable: a program transpiled for the project's python has to execute + /// on that python, or the checker's verdict and what actually runs are + /// answers about two different interpreters. + pub fn interpreter(&self, system: &dyn System) -> Option { + let sys_prefix = self.sys_prefix(); + // a virtual environment on windows keeps its executable in `Scripts`, + // while a system install has it directly at the prefix + let directories = if cfg!(windows) { + vec![sys_prefix.join("Scripts"), sys_prefix.to_path_buf()] + } else { + vec![sys_prefix.join("bin")] + }; + let names: &[&str] = if cfg!(windows) { + &["python.exe", "python3.exe"] + } else { + &["python3", "python"] + }; + + directories + .iter() + .flat_map(|directory| names.iter().map(|name| directory.join(name))) + .find(|candidate| system.is_file(candidate)) + } } /// Enumeration of the subdirectories of `sys.prefix` that could contain a diff --git a/crates/ty_static/src/env_vars.rs b/crates/ty_static/src/env_vars.rs index 22eb7ff0aa..97e9c820d6 100644 --- a/crates/ty_static/src/env_vars.rs +++ b/crates/ty_static/src/env_vars.rs @@ -79,6 +79,12 @@ impl EnvVars { /// Used to detect an activated virtual environment. pub const VIRTUAL_ENV: &'static str = "VIRTUAL_ENV"; + /// The interpreter `by run` executes the transpiled program on. + /// + /// Overrides the project environment, and is itself overridden by + /// `by run --python`. + pub const PYTHON: &'static str = "PYTHON"; + /// Adds additional directories to ty's search paths. /// The format is the same as the shell’s PATH: /// one or more directory pathnames separated by os appropriate pathsep diff --git a/docs/basedpython/getting-started.md b/docs/basedpython/getting-started.md index 5e46379f76..956eee282d 100644 --- a/docs/basedpython/getting-started.md +++ b/docs/basedpython/getting-started.md @@ -63,8 +63,7 @@ see [configuration](configuration.md) for everything that can go in there ## building -`by build` transpiles all `.by` files in the project and writes the output to -`out/`, mirroring the source structure: +`by build` writes the project to `out/` as python: ```sh by build @@ -77,6 +76,10 @@ utils.by -> out/utils.py build complete (2 files) ``` +that is the whole project, not only its `.by` files — a hand-written `.py` +module, a `py.typed`, a data file the program reads are all carried across to +the same place, so `out/` runs the way the source does + the generated `.py` files are ordinary python. run them with any python tool: ```sh @@ -86,6 +89,13 @@ mypy out/ ruff check out/ ``` +to ship the project rather than run it, build a wheel — see +[packaging](packaging.md): + +```sh +uv build +``` + ## CI integration ```yaml diff --git a/docs/basedpython/index.md b/docs/basedpython/index.md index 16bc458c40..2296f5010f 100644 --- a/docs/basedpython/index.md +++ b/docs/basedpython/index.md @@ -10,7 +10,7 @@ files any python tool can read pydantic, sqlalchemy, pytest and django are modelled directly, so the magic they do at runtime checks like ordinary code - **a build system** — write code against the latest version of python, and ship wheels that are compatible with old ones, no more waiting for 5 years to use something -- **basedpython, a python-like language that builds into python wheels** +- **basedpython, a python-like language that builds into python wheels** — `uv build`, and see [packaging](packaging.md) - **compiles into high performance python extension modules** - **a language server, formatter and linter** — high performance and feature rich tooling diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md new file mode 100644 index 0000000000..0230417113 --- /dev/null +++ b/docs/basedpython/packaging.md @@ -0,0 +1,189 @@ +# packaging + +a basedpython project builds into an ordinary python wheel. name the build +backend in `pyproject.toml`: + +```toml +[build-system] +requires = ["basedpython"] +build-backend = "basedpython.build" +``` + +and build it: + +```sh +uv build +``` + +that produces a wheel and a source distribution in `dist/`, publishable with +`uv publish` and installable by anyone, whether or not they have ever heard of +basedpython + +## starting from scratch + +`by init` writes a project already shaped this way: + +```sh +by init my-library --lib +``` + +```text +my-library/pyproject.toml +my-library/.python-version +my-library/README.md +my-library/src/my_library/__init__.by +``` + +leave off `--lib` and you also get a `main.by` and a configured entry point, so +`by run` works immediately + +## what a build produces + +`by build` writes the project to `out/` as python. that is the whole project, +not only its `.by` files: + +```text +src/app/main.by -> out/app/main.py +src/app/helper.py -> out/app/helper.py +src/app/settings.json -> out/app/settings.json +src/app/py.typed -> out/app/py.typed +``` + +a `.by` file is transpiled; everything else is carried across unchanged, to the +same place. the one rearrangement is the source root — `src/app/main.by` is the +module `app.main`, so it lands at `app/main.py` and not at `src/app/main.py` + +`out/` is a mirror, not a pile: what a previous build wrote and this one did not +is deleted, so a module you renamed does not go on being importable + +a stub stays a stub. `a.byi` builds to `a.pyi`, never to `a.py` + +### two sources, one module + +`main.by` and a hand-written `main.py` are both the module `main`, and a build +that quietly picked one would disagree with what python imports. so it says so: + +```text +`main.by` and `main.py` both build to `main.py` — they are the same module, so +one of them has to be renamed +``` + +## what a wheel carries + +the wheel holds the transpiled python, the `.by` sources beside it, and a +`by.typed` marker in each top-level package: + +```text +app/main.py +app/main.by +app/by.typed +``` + +python only ever imports the `.py`. the `.by` is there for the next basedpython +project along — see [depending on a basedpython +library](#depending-on-a-basedpython-library) + +to ship python only: + +```toml +[tool.basedpython.build] +sources = false +``` + +the marker still goes out either way. without the sources there is no `.by` for a +consumer to prefer, but the marker's contents are what declare which of this +project's dependencies are part of its interface — see +[declared dependencies](features/dependencies.md) + +## choosing what goes in + +`build.exclude` keeps files out, `build.include` narrows to a subset, and +exclusions win over inclusions: + +```toml +[tool.basedpython.build] +exclude = ["tests", "**/*.snapshot"] +``` + +`src.exclude` already bounds the build — a file the project excludes from itself +is not part of what it ships — so `build.exclude` is for the things that belong +in the project but not in the artifact + +caches, virtual environments, version-control directories, `target`, and the +output directory itself are excluded to begin with — by `src.exclude`, so a negation +there takes them back for the build too: + +```toml +[tool.basedpython.src] +exclude = ["!dist"] +``` + +## a version that lives in the source + +declare it dynamic and say where to read it from: + +```toml +[project] +dynamic = ["version"] + +[tool.basedpython.build] +version-from = "src/app/__init__.by" +``` + +```by +__version__ = "1.4.0" +``` + +## depending on a basedpython library + +a python project depends on a basedpython library the way it depends on any +other. nothing about the dependency is unusual: it is python in the wheel + +a *basedpython* project gets more. the `.by` sources travel with the wheel, and +the `by.typed` marker beside them says they are the authoritative surface — so +the declarations that have no python spelling survive the trip: + +```by +extension FlowContent: + def card(self) -> Div + +def load(path: str) -> Config raises ParseError +``` + +a consumer reading only the transpiled python sees `load` returning a `Config`. +a consumer reading the `.by` sees that it raises, and that `card` is available +on every `FlowContent` + +the marker is per package, and inherited by everything under it. it is the same +bargain [`py.typed`](https://peps.python.org/pep-0561/) strikes for inline +annotations: the package declares its own sources authoritative, rather than a +checker guessing + +## editable installs + +```sh +uv sync +``` + +installs the project pointing at `out/`, so `by build` is what refreshes an +editable install. run it after editing, the same way any compiled language +rebuilds before its changes are visible + +## a single-module project + +a wheel needs at least one importable package. a project whose only module is +`app.by` at the top level has nothing to package, and the build says so — move +it to `app/__init__.by` and it builds + +## running on the right python + +`by run` uses the project environment: the same interpreter `by check` resolves +imports against, which for a uv project is `.venv`. `$PYTHON` overrides it, and +`by run --python` overrides that + +a project that targets a newer python than the interpreter can run is reported +before anything executes: + +```text +this project targets python 3.13, but the interpreter this would run on is 3.9 +``` diff --git a/pyproject.toml b/pyproject.toml index 9a5fa921d3..6afdf773c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,11 +53,19 @@ Changelog = "https://github.com/KotlinIsland/basedpython/blob/main/CHANGELOG.md" # builds the standalone distribution crate, which produces both the `by` # (type-checker / transpiler) and `buff` (linter / formatter) binaries. maturin # packages every bin of the target crate as a console script, so the wheel ships -# both. no python module: a single `python -m basedpython` would be ambiguous -# between the two tools, so they're exposed purely as the `by` and `buff` scripts +# both. the two tools are exposed purely as those scripts — a single +# `python -m basedpython` would be ambiguous between them +# +# the python package alongside them is not an api: it is `basedpython.build`, the +# PEP 517 backend a basedpython project names in its own `[build-system]`. it has +# to be importable from the same distribution as `by`, because building a project +# means running `by` and a backend that found a different one on `PATH` would be +# transpiling with a version nobody chose [tool.maturin] bindings = "bin" manifest-path = "crates/basedpython/Cargo.toml" +python-source = "python" +module-name = "basedpython" strip = true [dependency-groups] diff --git a/python/basedpython/__init__.py b/python/basedpython/__init__.py new file mode 100644 index 0000000000..ebc7193dc4 --- /dev/null +++ b/python/basedpython/__init__.py @@ -0,0 +1,12 @@ +"""basedpython — a python-like language that builds into python wheels. + +The tools themselves are the `by` and `buff` executables this distribution +installs; there is no python api. What lives here is the build backend, so that +a basedpython project can name it in `[build-system]`: + + [build-system] + requires = ["basedpython"] + build-backend = "basedpython.build" +""" + +from __future__ import annotations diff --git a/python/basedpython/build.py b/python/basedpython/build.py new file mode 100644 index 0000000000..e50aefd414 --- /dev/null +++ b/python/basedpython/build.py @@ -0,0 +1,527 @@ +"""The PEP 517 build backend for basedpython projects. + + [build-system] + requires = ["basedpython"] + build-backend = "basedpython.build" + +A basedpython project is python once it is built, so this backend does the one +thing that makes it so — run `by build`, which transpiles the `.by` sources and +carries everything else across unchanged — and then hands the resulting tree to +`uv_build` to package. Splitting it there is deliberate: everything above the +transpile (core metadata, RECORD, wheel tags, entry points, editable installs) is +ordinary python packaging with nothing basedpython about it, and a backend that +reimplemented it would only be a second place for it to be subtly wrong. + +The wheel is plain python: the transpiled `.py` for python to import, the `.by` +sources beside them, and a `by.typed` marker saying the `.by` are the +authoritative surface. A python consumer sees a python library; a basedpython +consumer sees the declarations that have no python spelling. + +The source distribution keeps the `.by` untranspiled — it is the source — and is +written here rather than delegated, because no python backend knows that a `.by` +file is a source file. +""" + +from __future__ import annotations + +import io +import os +import re +import shutil +import subprocess +import sys +import sysconfig +import tarfile +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping, Sequence + +# the backend `uv_build` version this was written against. it is requested at +# build time rather than depended on outright: `by check` and `by run` are the +# overwhelming majority of what this distribution is used for, and neither of +# them packages anything +UV_BUILD_REQUIREMENT = "uv_build>=0.9,<10" + +# where `build_editable` stages the project. it is `by build`'s own default +# output directory on purpose: an editable install points python at this tree, so +# a plain `by build` is what refreshes an editable install +EDITABLE_STAGING_DIRECTORY = "out" + + +class BuildError(Exception): + """A build that cannot proceed, reported without a traceback into it.""" + + +# ── PEP 517 / PEP 660 hooks ────────────────────────────────────────────────── + + +def get_requires_for_build_wheel( + config_settings: Mapping[str, Any] | None = None, +) -> list[str]: + return _requirements() + + +def get_requires_for_build_sdist( + config_settings: Mapping[str, Any] | None = None, +) -> list[str]: + return _requirements() + + +def get_requires_for_build_editable( + config_settings: Mapping[str, Any] | None = None, +) -> list[str]: + return _requirements() + + +def prepare_metadata_for_build_wheel( + metadata_directory: str, + config_settings: Mapping[str, Any] | None = None, +) -> str: + with _staged() as staging: + return _delegate( + "prepare_metadata_for_build_wheel", + staging, + metadata_directory, + config_settings, + ) + + +def prepare_metadata_for_build_editable( + metadata_directory: str, + config_settings: Mapping[str, Any] | None = None, +) -> str: + with _staged() as staging: + return _delegate( + "prepare_metadata_for_build_editable", + staging, + metadata_directory, + config_settings, + ) + + +def build_wheel( + wheel_directory: str, + config_settings: Mapping[str, Any] | None = None, + metadata_directory: str | None = None, +) -> str: + with _staged() as staging: + return _delegate("build_wheel", staging, wheel_directory, config_settings) + + +def build_editable( + wheel_directory: str, + config_settings: Mapping[str, Any] | None = None, + metadata_directory: str | None = None, +) -> str: + # an editable install is a pointer at a directory, so the directory has to + # outlive the build. it is the same one `by build` writes, which is what + # makes the install editable at all: transpiled python is what gets imported, + # and re-running the build is what updates it + staging = Path.cwd() / EDITABLE_STAGING_DIRECTORY + built = _stage(staging) + _write_staged_pyproject(staging, built) + return _delegate("build_editable", staging, wheel_directory, config_settings) + + +def build_sdist( + sdist_directory: str, + config_settings: Mapping[str, Any] | None = None, +) -> str: + project_root = Path.cwd() + metadata = _read_project_metadata(project_root) + name = _normalized_name(metadata["name"]) + version = metadata["version"] + directory_name = f"{name}-{version}" + + with tempfile.TemporaryDirectory() as scratch: + staging = Path(scratch) / "build" + # one build answers both halves of a source distribution: what the + # project is made of, and — through the metadata the wheel would carry — + # what to say about it + built = _stage(staging) + _write_staged_pyproject(staging, built) + metadata_directory = Path(scratch) / "metadata" + metadata_directory.mkdir() + dist_info = _delegate( + "prepare_metadata_for_build_wheel", staging, str(metadata_directory), None + ) + pkg_info = (metadata_directory / dist_info / "METADATA").read_bytes() + + sdist = Path(sdist_directory) + sdist.mkdir(parents=True, exist_ok=True) + target = sdist / f"{directory_name}.tar.gz" + with tarfile.open(target, "w:gz", format=tarfile.PAX_FORMAT) as archive: + for source in built.sources: + if not (project_root / source).is_file(): + continue + archive.add( + project_root / source, + arcname=f"{directory_name}/{source}", + recursive=False, + ) + info = tarfile.TarInfo(f"{directory_name}/PKG-INFO") + info.size = len(pkg_info) + archive.addfile(info, io.BytesIO(pkg_info)) + + return target.name + + +# ── staging ────────────────────────────────────────────────────────────────── + + +@contextmanager +def _staged() -> Iterator[Path]: + """The project, built as python, in a directory that lasts for one hook.""" + with tempfile.TemporaryDirectory() as directory: + staging = Path(directory) / "build" + built = _stage(staging) + _write_staged_pyproject(staging, built) + yield staging + + +class Staged: + """What a build read, and what it produced.""" + + __slots__ = ("packages", "sources") + + def __init__(self, sources: list[str], packages: list[str]) -> None: + self.sources = sources + self.packages = packages + + +def _stage(staging: Path) -> Staged: + """Build the project into `staging`, and report what came of it. + + Which files the project is made of, and which packages it builds into, are + both the build's answers rather than this backend's. Reading a staged tree + back to guess at them would be a second answer to keep in step — and it would + guess wrong, since a directory in the output is not necessarily something the + project ships. + """ + sources: list[str] = [] + packages: list[str] = [] + for line in _run_by( + "build", "--out", str(staging), "--print-manifest" + ).splitlines(): + kind, _, value = line.strip().partition(" ") + if kind == "input": + sources.append(value) + elif kind == "package": + packages.append(value) + return Staged(sorted(set(sources)), sorted(set(packages))) + + +def _write_staged_pyproject(staging: Path, built: Staged) -> None: + """Describe the staged tree to the backend that packages it. + + The project's own `pyproject.toml` was copied across by the build, and it + names *this* backend — handing it back unchanged would build the project + again, forever. So it is replaced by one describing what the staged tree + actually is: plain python, packages at the top level, and a version that is + settled rather than dynamic. + """ + project_root = Path.cwd() + metadata = _read_project_metadata(project_root) + if not built.packages: + raise BuildError( + "this project has no package to build a wheel from — a wheel needs at " + "least one importable package, so a top-level module like `app.by` has " + "to become `app/__init__.by`" + ) + + document = { + "build-system": { + "requires": [UV_BUILD_REQUIREMENT], + "build-backend": "uv_build", + }, + "project": metadata, + "tool": { + "uv": { + "build-backend": { + # the staged tree *is* the module root: `by build` already + # resolved the project's layout, so `src/pkg` arrives as `pkg` + "module-root": "", + "module-name": built.packages, + } + } + }, + } + (staging / "pyproject.toml").write_text(_toml(document), encoding="utf-8") + + +# ── delegation ─────────────────────────────────────────────────────────────── + + +def _delegate( + hook_name: str, + staging: Path, + out_directory: str, + config_settings: Mapping[str, Any] | None, +) -> str: + try: + import uv_build + except ImportError as error: # pragma: no cover - only without build isolation + raise BuildError( + f"packaging a basedpython project needs `{UV_BUILD_REQUIREMENT}`, which " + "is normally installed into the build environment automatically. it is " + "missing, which usually means the build was run with isolation disabled" + ) from error + + hook = getattr(uv_build, hook_name) + out = os.path.abspath(out_directory) + Path(out).mkdir(parents=True, exist_ok=True) + previous = Path.cwd() + os.chdir(staging) + try: + with _scripts_on_path(): + return hook(out, config_settings) + finally: + os.chdir(previous) + + +@contextmanager +def _scripts_on_path() -> Iterator[None]: + """Make this environment's console scripts findable by name. + + `uv_build` shells out to a `uv-build` executable it finds on `PATH`. A + frontend is supposed to put the build environment's scripts there, and they + all do — but the cost of not relying on it is one line. + """ + scripts = sysconfig.get_path("scripts") + previous = os.environ.get("PATH", "") + os.environ["PATH"] = os.pathsep.join(filter(None, (scripts, previous))) + try: + yield + finally: + os.environ["PATH"] = previous + + +def _requirements() -> list[str]: + requirements = [UV_BUILD_REQUIREMENT] + if sys.version_info < (3, 11): + requirements.append("tomli>=2") + return requirements + + +# ── the project's own metadata ─────────────────────────────────────────────── + + +def _read_project_metadata(project_root: Path) -> dict[str, Any]: + """The `[project]` table, with anything dynamic settled. + + It is carried into the staged tree verbatim, because it is what becomes the + wheel's core metadata. The one thing that cannot be carried is a dynamic + version: the backend downstream has no way to compute one, and the place the + version actually lives — a `.by` module — is not something it can read. So it + is resolved here, from the python that module was transpiled into. + """ + pyproject = project_root / "pyproject.toml" + if not pyproject.is_file(): + raise BuildError("a basedpython project needs a `pyproject.toml`") + + document = _parse_toml(pyproject.read_bytes()) + metadata = dict(document.get("project", {})) + if not metadata: + raise BuildError("`pyproject.toml` has no `[project]` table to build from") + + dynamic = list(metadata.get("dynamic", [])) + if "version" in dynamic: + metadata["version"] = _dynamic_version(project_root, document) + dynamic.remove("version") + if dynamic: + raise BuildError( + "this backend cannot resolve dynamic metadata " + f"{', '.join(sorted(dynamic))} — declare it in `[project]` instead" + ) + metadata.pop("dynamic", None) + if "version" not in metadata: + raise BuildError("`[project]` has neither a `version` nor a dynamic one") + return metadata + + +def _dynamic_version(project_root: Path, document: Mapping[str, Any]) -> str: + """Read `__version__` out of the module the project points at.""" + configured = ( + document.get("tool", {}) + .get("basedpython", {}) + .get("build", {}) + .get("version-from") + ) + if not configured: + raise BuildError( + '`[project] dynamic = ["version"]` needs somewhere to read the version ' + 'from — set `[tool.basedpython.build] version-from = "src/pkg/__init__.by"`' + ) + source = project_root / configured + if not source.is_file(): + raise BuildError( + f"`build.version-from` points at `{configured}`, which is not a file" + ) + + for line in source.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + # `__version__ = "1.2.3"`, however it is spelled around the assignment. + # reading the text rather than importing keeps this from executing a + # module that has not been transpiled yet + if stripped.startswith("__version__"): + _, _, value = stripped.partition("=") + value = value.strip().split("#")[0].strip() + if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0]: + return value[1:-1] + raise BuildError(f"`{configured}` has no `__version__` to read the version from") + + +def _parse_toml(contents: bytes) -> dict[str, Any]: + try: + import tomllib + except ImportError: # python < 3.11 + import tomli as tomllib # type: ignore[no-redef] + return tomllib.loads(contents.decode("utf-8")) + + +def _normalized_name(name: str) -> str: + return re.sub(r"[-_.]+", "_", name).lower() + + +# ── writing the staged pyproject ───────────────────────────────────────────── + + +def _toml(document: Mapping[str, Any]) -> str: + """Render a mapping as TOML. + + Only what a `[project]` table holds has to survive this: strings, numbers, + booleans, arrays, tables, and arrays of tables. That is the whole of PEP 621, + and rendering it here is what lets the project's metadata reach the packaging + backend unaltered. + """ + lines: list[str] = [] + _render_table(document, [], lines) + return "\n".join(lines) + "\n" + + +def _render_table( + table: Mapping[str, Any], path: Sequence[str], lines: list[str] +) -> None: + scalars = {key: value for key, value in table.items() if not _is_table_like(value)} + tables = {key: value for key, value in table.items() if _is_table_like(value)} + + if path and (scalars or not tables): + lines.append(f"[{_render_key_path(path)}]") + for key, value in scalars.items(): + lines.append(f"{_render_key(key)} = {_render_value(value)}") + if path and (scalars or not tables): + lines.append("") + + for key, value in tables.items(): + nested = [*path, key] + if isinstance(value, dict): + _render_table(value, nested, lines) + else: + for element in value: + lines.append(f"[[{_render_key_path(nested)}]]") + for inner_key, inner_value in element.items(): + lines.append( + f"{_render_key(inner_key)} = {_render_value(inner_value)}" + ) + lines.append("") + + +def _is_table_like(value: Any) -> bool: + if isinstance(value, dict): + return True + # an array of tables is rendered as `[[name]]` blocks; an array of anything + # else is an ordinary inline array + return ( + isinstance(value, list) + and len(value) > 0 + and all(isinstance(element, dict) for element in value) + ) + + +def _render_key_path(path: Sequence[str]) -> str: + return ".".join(_render_key(part) for part in path) + + +def _render_key(key: str) -> str: + if key and all(character.isalnum() or character in "-_" for character in key): + return key + return _render_string(key) + + +def _render_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, str): + return _render_string(value) + if isinstance(value, (int, float)): + return repr(value) + if isinstance(value, dict): + rendered = ", ".join( + f"{_render_key(key)} = {_render_value(inner)}" + for key, inner in value.items() + ) + return "{" + rendered + "}" + if isinstance(value, (list, tuple)): + rendered = ", ".join(_render_value(element) for element in value) + return "[" + rendered + "]" + raise BuildError( + f"`pyproject.toml` holds a value this backend cannot carry over: {value!r}" + ) + + +def _render_string(value: str) -> str: + escaped = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + return f'"{escaped}"' + + +# ── the `by` executable ────────────────────────────────────────────────────── + + +def _run_by(*arguments: str) -> str: + executable = _by_executable() + completed = subprocess.run( + [executable, *arguments], + stdout=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise BuildError( + f"`by {' '.join(arguments)}` failed with exit code {completed.returncode}" + ) + return completed.stdout.decode("utf-8") + + +def _by_executable() -> str: + """Find the `by` that belongs to this installation. + + It ships in the same distribution as this module, so it is in this + environment's scripts directory. Looking there first rather than on `PATH` + means the build uses the transpiler it was installed with, not whichever one + happens to be earlier in the shell's path. + """ + name = "by.exe" if os.name == "nt" else "by" + candidates = [ + Path(sysconfig.get_path("scripts")) / name, + Path(sys.executable).parent / name, + ] + for candidate in candidates: + if candidate.is_file(): + return str(candidate) + found = shutil.which(name) + if found: + return found + raise BuildError( + "could not find the `by` executable, which is what builds a basedpython " + "project. it is installed by the `basedpython` distribution — check that " + "`[build-system] requires` names it" + ) diff --git a/scripts/test_build_backend.py b/scripts/test_build_backend.py new file mode 100644 index 0000000000..cad15370cb --- /dev/null +++ b/scripts/test_build_backend.py @@ -0,0 +1,274 @@ +"""Tests for the basedpython PEP 517 build backend. + +Run with:: + + uv run --no-project --with pytest pytest scripts/test_build_backend.py + +What is covered here is everything the backend decides on its own: what the +staged `pyproject.toml` says, how a project's metadata survives being written +back out, and where a dynamic version comes from. The packaging itself is +`uv_build`'s, and the transpile is `by`'s; neither is reimplemented here, so +neither is tested here. +""" + +from __future__ import annotations + +import os +import sys +import tomllib +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent / "python")) + +from basedpython.build import ( + BuildError, + Staged, + _read_project_metadata, + _toml, + _write_staged_pyproject, +) + + +def render_and_parse(document: dict) -> dict: + """A document, through the writer and back.""" + return tomllib.loads(_toml(document)) + + +# ── the TOML the staged tree is described by ───────────────────────────────── + + +def test_scalars_survive_the_round_trip() -> None: + document = { + "project": { + "name": "thing", + "version": "1.2.3", + "requires-python": ">=3.12", + "keywords": ["a", "b"], + } + } + assert render_and_parse(document) == document + + +def test_a_nested_table_survives_the_round_trip() -> None: + document = { + "project": { + "name": "thing", + "version": "1.0.0", + "urls": {"Repository": "https://example.com/thing"}, + "optional-dependencies": {"dev": ["pytest>=8"]}, + } + } + assert render_and_parse(document) == document + + +def test_an_array_of_tables_survives_the_round_trip() -> None: + document = { + "project": { + "name": "thing", + "version": "1.0.0", + "authors": [ + {"name": "A Person", "email": "a@example.com"}, + {"name": "Another"}, + ], + } + } + assert render_and_parse(document) == document + + +def test_a_key_that_is_not_bare_is_quoted() -> None: + """`[project.entry-points."my.group"]` is an ordinary shape, and a key with a + dot in it splits into two tables unless it is quoted.""" + document = { + "project": { + "name": "thing", + "version": "1.0.0", + "entry-points": {"my.group": {"plugin": "thing:main"}}, + } + } + assert render_and_parse(document) == document + + +def test_a_string_that_needs_escaping_survives_the_round_trip() -> None: + document = { + "project": { + "name": "thing", + "version": "1.0.0", + "description": 'a "quoted" \\ backslash\nand a newline', + } + } + assert render_and_parse(document) == document + + +def test_booleans_and_numbers_are_rendered_as_themselves() -> None: + document = {"tool": {"example": {"flag": True, "count": 3}}} + assert render_and_parse(document) == document + + +def test_an_empty_array_stays_an_array() -> None: + """`dependencies = []` is not an array of tables, however it looks.""" + document = {"project": {"name": "thing", "version": "1.0.0", "dependencies": []}} + assert render_and_parse(document) == document + + +def test_a_value_that_cannot_be_carried_over_is_reported() -> None: + with pytest.raises(BuildError, match="cannot carry over"): + _toml({"project": {"name": "thing", "version": object()}}) + + +# ── reading the project's own metadata ─────────────────────────────────────── + + +def write_project(directory: Path, pyproject: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + (directory / "pyproject.toml").write_text(pyproject, encoding="utf-8") + return directory + + +def test_the_project_table_is_carried_over_whole(tmp_path: Path) -> None: + root = write_project( + tmp_path, + """ + [build-system] + requires = ["basedpython"] + build-backend = "basedpython.build" + + [project] + name = "thing" + version = "1.0.0" + dependencies = ["packaging>=24"] + """, + ) + metadata = _read_project_metadata(root) + assert metadata == { + "name": "thing", + "version": "1.0.0", + "dependencies": ["packaging>=24"], + } + + +def test_a_project_without_a_pyproject_is_reported(tmp_path: Path) -> None: + with pytest.raises(BuildError, match=r"needs a \`pyproject\.toml\`"): + _read_project_metadata(tmp_path) + + +def test_a_pyproject_without_a_project_table_is_reported(tmp_path: Path) -> None: + root = write_project(tmp_path, '[build-system]\nrequires = ["basedpython"]\n') + with pytest.raises(BuildError, match="no `\\[project\\]` table"): + _read_project_metadata(root) + + +def test_a_dynamic_version_is_read_from_the_module_it_points_at(tmp_path: Path) -> None: + root = write_project( + tmp_path, + """ + [project] + name = "thing" + dynamic = ["version"] + + [tool.basedpython.build] + version-from = "src/thing/__init__.by" + """, + ) + module = root / "src" / "thing" + module.mkdir(parents=True) + (module / "__init__.by").write_text( + '"""a docstring"""\n\n__version__ = "4.5.6" # the one that counts\n', + encoding="utf-8", + ) + + metadata = _read_project_metadata(root) + assert metadata["version"] == "4.5.6" + assert "dynamic" not in metadata + + +def test_a_dynamic_version_with_nowhere_to_read_it_from_is_reported( + tmp_path: Path, +) -> None: + root = write_project( + tmp_path, + '[project]\nname = "thing"\ndynamic = ["version"]\n', + ) + with pytest.raises(BuildError, match="version-from"): + _read_project_metadata(root) + + +def test_a_version_source_without_a_version_is_reported(tmp_path: Path) -> None: + root = write_project( + tmp_path, + """ + [project] + name = "thing" + dynamic = ["version"] + + [tool.basedpython.build] + version-from = "empty.by" + """, + ) + (root / "empty.by").write_text("x = 1\n", encoding="utf-8") + with pytest.raises(BuildError, match="no `__version__`"): + _read_project_metadata(root) + + +def test_other_dynamic_metadata_is_reported_rather_than_dropped(tmp_path: Path) -> None: + """Silently dropping it would ship a wheel missing what the project declared.""" + root = write_project( + tmp_path, + '[project]\nname = "thing"\nversion = "1.0.0"\ndynamic = ["readme"]\n', + ) + with pytest.raises(BuildError, match="readme"): + _read_project_metadata(root) + + +# ── describing the staged tree to the backend that packages it ─────────────── + + +def staged_pyproject(tmp_path: Path, pyproject: str, built: Staged) -> dict: + """The document written into the staging directory for a project.""" + root = write_project(tmp_path / "project", pyproject) + staging = tmp_path / "staging" + staging.mkdir() + + previous = Path.cwd() + os.chdir(root) + try: + _write_staged_pyproject(staging, built) + finally: + os.chdir(previous) + return tomllib.loads((staging / "pyproject.toml").read_text(encoding="utf-8")) + + +PROJECT = """ +[build-system] +requires = ["basedpython"] +build-backend = "basedpython.build" + +[project] +name = "thing" +version = "1.0.0" +""" + + +def test_the_staged_project_is_packaged_by_the_delegate(tmp_path: Path) -> None: + """The project's own `pyproject.toml` names this backend. Handed back + unchanged it would build the project again, forever.""" + document = staged_pyproject( + tmp_path, PROJECT, Staged(sources=[], packages=["thing"]) + ) + assert document["build-system"]["build-backend"] == "uv_build" + assert document["project"]["name"] == "thing" + + +def test_the_staged_tree_is_its_own_module_root(tmp_path: Path) -> None: + """`by build` already resolved the layout, so `src/pkg` arrives as `pkg`.""" + document = staged_pyproject( + tmp_path, PROJECT, Staged(sources=[], packages=["alpha", "beta"]) + ) + assert document["tool"]["uv"]["build-backend"]["module-root"] == "" + assert document["tool"]["uv"]["build-backend"]["module-name"] == ["alpha", "beta"] + + +def test_a_project_with_no_package_to_ship_is_reported(tmp_path: Path) -> None: + with pytest.raises(BuildError, match="no package to build a wheel from"): + staged_pyproject(tmp_path, PROJECT, Staged(sources=[], packages=[])) diff --git a/ty.schema.json b/ty.schema.json index d0a40499eb..b17aa6a38b 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -13,6 +13,17 @@ } ] }, + "build": { + "description": "Configures what `by build` writes, and what a wheel of this project carries.", + "anyOf": [ + { + "$ref": "#/definitions/BuildOptions" + }, + { + "type": "null" + } + ] + }, "editor": { "description": "Configures the parts of the editor experience that type checking does not decide.", "anyOf": [ @@ -253,6 +264,52 @@ "$ref": "#/definitions/string" } }, + "BuildOptions": { + "type": "object", + "properties": { + "exclude": { + "description": "Files to keep out of the build output.\n\nThe syntax is the same as `src.exclude`, and paths are anchored to the\nproject root. Excluding a `.by` file keeps its transpiled output out of\nthe build as well.", + "anyOf": [ + { + "$ref": "#/definitions/Array_of_string" + }, + { + "type": "null" + } + ] + }, + "include": { + "description": "Files to carry into the build output verbatim, in addition to the ones\nthat are there by default.\n\n`by build` mirrors the whole module tree: a `.by` file is transpiled, and\nevery other file — a hand-written `.py`, a `py.typed` marker, a template,\na data file — is copied to the same place in the output. `include` is for\nthe files that sit *outside* a module root and still belong in the build,\nsuch as a data directory next to `src`.\n\nThe syntax is the same as `src.include`, and paths are anchored to the\nproject root. `exclude` takes precedence over `include`.", + "anyOf": [ + { + "$ref": "#/definitions/Array_of_string" + }, + { + "type": "null" + } + ] + }, + "sources": { + "description": "Whether the build output carries the `.by` sources alongside the python\nthey were transpiled into, with a `by.typed` marker naming them as the\nauthoritative surface.\n\nThis is what lets one basedpython project depend on another: a downstream\npython project reads the transpiled `.py` and is served perfectly, while a\ndownstream basedpython project reads the `.by` and keeps the declarations\nthat have no python spelling — `extension` blocks, `raises` clauses,\nread-only `let`, sum types.\n\nEnabled by default. Turn it off to ship python only.", + "type": [ + "boolean", + "null" + ] + }, + "version-from": { + "description": "The module to read `__version__` from, when `[project]` declares\n`dynamic = [\"version\"]`.\n\nThis is read when a wheel or a source distribution is built, not by the\nchecker: a version has to be settled before the packaging backend sees the\nproject, and the place it lives is a `.by` module that backend cannot\nread.\n\nThe value is a path relative to the project root.", + "anyOf": [ + { + "$ref": "#/definitions/string" + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false + }, "CommonAliases": { "type": "object", "additionalProperties": { diff --git a/zensical.toml b/zensical.toml index 4dbab0b31a..f7880608ff 100644 --- a/zensical.toml +++ b/zensical.toml @@ -68,6 +68,9 @@ basedpython = "index.md" [[project.nav]] "getting started" = "getting-started.md" +[[project.nav]] +packaging = "packaging.md" + [[project.nav]] configuration = "configuration.md" From 972c52034b26807048de69ec3150404b1b49f876 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:13:46 +1000 Subject: [PATCH 2/4] a wheel declares what lowering needed to build it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lowering for an older python can put a name in the output that only `typing_extensions` has there — `Self` on 3.9. the project never asked for it, so it cannot have declared it, and the wheel shipped without it: it installed cleanly and failed on the first import of the module that borrowed the name. the transpile is what knows, so the transpile is what reports it, and only when it actually reached for it — the same project built for a python that has the name in its own `typing` gets no such dependency. a project that already names the distribution keeps its own constraint. --- crates/by_transforms/src/lib.rs | 91 +++++++++++++++++++++++++++++---- crates/ty/src/by_commands.rs | 25 +++++++-- crates/ty/tests/by_e2e.rs | 52 +++++++++++++++++++ docs/basedpython/packaging.md | 16 ++++++ python/basedpython/build.py | 57 ++++++++++++++++++--- scripts/test_build_backend.py | 49 ++++++++++++++++++ 6 files changed, 269 insertions(+), 21 deletions(-) diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 68ba0e6ee7..d186c75d25 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -123,8 +123,17 @@ fn run_erased_union_phase<'a>( /// type-aware passes see only this file). Used for stdin input and tests; the /// file-backed [`transpile_typed`] resolves cross-module types. pub fn transpile(source: &str, config: &Config) -> Result { + transpile_with_report(source, config).map(|(output, _)| output) +} + +/// Like [`transpile`], and also reports what the emitted python needs installed +/// to run. +pub fn transpile_with_report( + source: &str, + config: &Config, +) -> Result<(String, RuntimeRequirements), String> { if config.is_python { - return Ok(source.to_owned()); + return Ok((source.to_owned(), RuntimeRequirements::default())); } // one db over the original source, shared by the qualification phase below @@ -197,7 +206,7 @@ pub fn transpile(source: &str, config: &Config) -> Result { } // --- Phase 2: import-redirect, surface-syntax cleanup, lazy-import marking --- - let final_output = run_import_redirect_phase(output, config); + let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &model.eagerly_imported_modules()); @@ -207,7 +216,7 @@ pub fn transpile(source: &str, config: &Config) -> Result { verify_syntax(&final_output).map_err(|e| e.message)?; verify_target_syntax(&final_output, config).map_err(|e| e.message)?; - Ok(final_output) + Ok((final_output, requirements)) } /// Transpile using ty's full type inference. `db` and `file` must already @@ -243,12 +252,71 @@ pub fn transpile_typed_with_map( config: &Config, rebuild: Option>, ) -> Result<(String, Vec>), TranspileError> { + transpile_typed_with_report(db, file, config, rebuild) + .map(|(output, line_map, _)| (output, line_map)) +} + +/// What the emitted python needs at run time that the standard library does not +/// provide. +/// +/// Lowering for an older python can put a name in the output that only +/// `typing_extensions` has there — `Self` on 3.9, say. That is a real dependency +/// of the built artifact, and nothing in the source says so, which is why the +/// transpile is what reports it: a wheel that shipped without it would install +/// cleanly and fail on the first import. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct RuntimeRequirements { + typing_extensions: bool, +} + +/// What the emitted python needs when lowering reached for `typing_extensions`. +/// +/// One floor for every name rather than one per name: which release of +/// `typing_extensions` first carried each backport is not something the emitted +/// code records, and a floor too low is a wheel that installs and then fails on +/// an import. It has to cover every name +/// [`ty_python_semantic::basedpython_typing_added_in`] can redirect, which is why +/// it lives beside the pass that does the redirecting rather than beside the +/// command that reports it. +const TYPING_EXTENSIONS_REQUIREMENT: &str = "typing_extensions>=4.12"; + +impl RuntimeRequirements { + /// Fold in what another module needed. + pub fn merge(&mut self, other: Self) { + self.typing_extensions |= other.typing_extensions; + } + + /// The requirements, spelled the way a `[project] dependencies` entry is. + /// + /// A list rather than a set of flags, so that a second requirement is a line + /// here instead of an edit at every place a caller asks what is needed. + pub fn specifiers(self) -> Vec<&'static str> { + let mut specifiers = Vec::new(); + if self.typing_extensions { + specifiers.push(TYPING_EXTENSIONS_REQUIREMENT); + } + specifiers + } +} + +/// Like [`transpile_typed_with_map`], and also reports what the emitted python +/// needs installed to run. +pub fn transpile_typed_with_report( + db: &dyn ty_python_semantic::Db, + file: File, + config: &Config, + rebuild: Option>, +) -> Result<(String, Vec>, RuntimeRequirements), TranspileError> { let source_ref = ruff_db::source::source_text(db, file); let original_source = source_ref.as_str(); if config.is_python { let out = original_source.to_owned(); - return Ok((out, source_map::line_table(original_source, &[]))); + return Ok(( + out, + source_map::line_table(original_source, &[]), + RuntimeRequirements::default(), + )); } // erased-union reification: give a `list[int] | list[str]` parameter a @@ -364,7 +432,7 @@ pub fn transpile_typed_with_map( return Err(first.clone().into()); } - let final_output = run_import_redirect_phase(output, config); + let (final_output, requirements) = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &eager_imports); let final_output = run_version_polyfill_phase(final_output, config); @@ -410,7 +478,7 @@ pub fn transpile_typed_with_map( return Err(err); } - Ok((final_output, line_map)) + Ok((final_output, line_map, requirements)) } fn newline_count(s: &str) -> usize { @@ -499,7 +567,7 @@ fn run_anon_named_tuple_cleanup(mut source: String, config: &Config) -> Result String { +fn run_import_redirect_phase(source: String, config: &Config) -> (String, RuntimeRequirements) { let (db, file) = make_in_memory_db(&source); let source_ref = ruff_db::source::source_text(&db, file); let src = source_ref.as_str(); @@ -515,11 +583,16 @@ fn run_import_redirect_phase(source: String, config: &Config) -> String { } if typing_redirect.edits.is_empty() { - return source; + return (source, RuntimeRequirements::default()); } let (output, _) = apply_transforms_once(src, typing_redirect.edits); - output + ( + output, + RuntimeRequirements { + typing_extensions: true, + }, + ) } /// Lazy-import marking phase: walks the post-typing-redirect output and diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 5dcd337348..87937e241b 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -206,6 +206,7 @@ pub(crate) fn cmd_run( &config, CheckGate::AllErrors, &rebuilder, + &mut by_transforms::RuntimeRequirements::default(), |emitted| { let relative = transpiled_destination(&roots, &root, emitted.by_path); traceback_entries.push(stage_module(&mut staging, &relative, emitted)?); @@ -715,12 +716,14 @@ pub(crate) fn cmd_build( // the directory where a `.by` really can be saved after the transpile, which // is what the digests beside the map are for let mut entries: Vec = Vec::new(); + let mut requirements = by_transforms::RuntimeRequirements::default(); if !render_check_and_transpile( &db, &handles, &config, CheckGate::ParseErrorsOnly, &rebuilder, + &mut requirements, |emitted| { let relative = transpiled_destination(&roots, &root, emitted.by_path); let entry = stage_module(&mut staging, &relative, emitted)?; @@ -740,7 +743,7 @@ pub(crate) fn cmd_build( stage_by_typed_markers(&db, &mut staging, &roots, &root)?; write_sourcemap_module(&mut staging, &entries)?; if print_manifest { - print_build_manifest(&staging, &roots, &root)?; + print_build_manifest(&staging, &roots, &root, requirements)?; } staging.finish()?; @@ -756,7 +759,12 @@ pub(crate) fn cmd_build( /// out. Answering them here rather than in the packaging layer keeps one answer /// to "what is this project", instead of a second one that has to be kept in step. #[allow(clippy::print_stdout)] -fn print_build_manifest(staging: &Staging, roots: &[PathBuf], root: &Path) -> anyhow::Result<()> { +fn print_build_manifest( + staging: &Staging, + roots: &[PathBuf], + root: &Path, + requirements: by_transforms::RuntimeRequirements, +) -> anyhow::Result<()> { use std::io::Write as _; let mut stdout = io::stdout().lock(); @@ -767,6 +775,12 @@ fn print_build_manifest(staging: &Staging, roots: &[PathBuf], root: &Path) -> an for package in staged_packages(staging, roots, root) { writeln!(stdout, "package {package}")?; } + // lowering for an older python can put a name in the output that only + // `typing_extensions` has there. nothing in the source says so, so nothing + // but the build can + for specifier in requirements.specifiers() { + writeln!(stdout, "requires {specifier}")?; + } Ok(()) } @@ -1258,6 +1272,7 @@ fn forward_dir(dir: &Path, config: &Config) -> anyhow::Result { config, CheckGate::ParseErrorsOnly, &rebuilder, + &mut by_transforms::RuntimeRequirements::default(), |emitted| { let py = emitted.by_path.with_extension("py"); fs::write(&py, emitted.python).with_context(|| format!("{}", py.display()))?; @@ -1787,6 +1802,7 @@ fn render_check_and_transpile( config: &Config, gate: CheckGate, rebuilder: &Rebuilder, + requirements: &mut by_transforms::RuntimeRequirements, mut consume: impl FnMut(&Transpiled<'_>) -> anyhow::Result<()>, ) -> anyhow::Result { let mut all_diagnostics: Vec = Vec::new(); @@ -1824,8 +1840,9 @@ fn render_check_and_transpile( if unusable.contains(file) { continue; } - match by_transforms::transpile_typed_with_map(db, *file, config, Some(&rebuild)) { - Ok((out, line_map)) => { + match by_transforms::transpile_typed_with_report(db, *file, config, Some(&rebuild)) { + Ok((out, line_map, needed)) => { + requirements.merge(needed); let by_source = source_text(db, *file); consume(&Transpiled { by_path: bpy, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index b305660942..e9b7618c54 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3269,6 +3269,58 @@ fn build_ships_a_source_directory_that_is_itself_a_package() { assert!(dir.path().join("out/src/mymod/__init__.py").exists()); } +/// lowering for an older python can put a name in the output that only +/// `typing_extensions` has there. nothing in the source says so — the project +/// never asked for it — so nothing but the build can, and a wheel that shipped +/// without it would install cleanly and fail on the first import +#[test] +fn build_reports_what_lowering_needs_at_run_time() { + let dir = tempfile::tempdir().expect("tempdir"); + let package = dir.path().join("src").join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"app\"\nversion = \"0.1.0\"\nrequires-python = \">=3.9\"\n", + ) + .unwrap(); + // `Self` reached `typing` in 3.11, so a 3.9 target has to borrow it + fs::write( + package.join("__init__.by"), + "from typing import Self\n\nclass N:\n def me(self) -> Self:\n return self\n", + ) + .unwrap(); + + let manifest = |extra: &[&str]| -> String { + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--print-manifest"]) + .args(extra) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + assert!( + output.status.success(), + "by build failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() + }; + + let lowered = manifest(&[]); + assert!( + lowered + .lines() + .any(|line| line.starts_with("requires typing_extensions")), + "a 3.9 target borrows the name, so the wheel depends on it:\n{lowered}" + ); + + // and on a python that has it, the dependency would be dead weight + let native = manifest(&["--min-version", "3.13"]); + assert!( + !native.contains("requires "), + "a 3.13 target needs no backport:\n{native}" + ); +} + // ── running a project, not just its `.by` files ────────────────────────────── /// the same hole at run time, where it is fatal rather than untidy: `by run` diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index 0230417113..25ddc67850 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -118,6 +118,22 @@ there takes them back for the build too: exclude = ["!dist"] ``` +## dependencies lowering adds + +Building for an older python can put a name in the output that only +`typing_extensions` has there — `Self` on 3.9, say. The project never asked for +it, so it cannot have declared it, and a wheel that shipped without it would +install cleanly and fail on the first import. The build reports what it reached +for and the wheel declares it: + +```text +Requires-Dist: typing-extensions>=4.12 +``` + +Only when lowering actually needed it. The same project built for a python that +has the name in its own `typing` gets no such dependency. A project that already +names the distribution keeps its own constraint + ## a version that lives in the source declare it dynamic and say where to read it from: diff --git a/python/basedpython/build.py b/python/basedpython/build.py index e50aefd414..8cbac2be18 100644 --- a/python/basedpython/build.py +++ b/python/basedpython/build.py @@ -184,16 +184,22 @@ def _staged() -> Iterator[Path]: class Staged: - """What a build read, and what it produced.""" + """What a build read, what it produced, and what the result needs to run.""" - __slots__ = ("packages", "sources") + __slots__ = ("packages", "requires", "sources") - def __init__(self, sources: list[str], packages: list[str]) -> None: + def __init__( + self, + sources: list[str], + packages: list[str], + requires: list[str] | None = None, + ) -> None: self.sources = sources self.packages = packages + self.requires = requires or [] -def _stage(staging: Path) -> Staged: +def _stage(staging: Path, python_version: str | None = None) -> Staged: """Build the project into `staging`, and report what came of it. Which files the project is made of, and which packages it builds into, are @@ -202,17 +208,22 @@ def _stage(staging: Path) -> Staged: guess wrong, since a directory in the output is not necessarily something the project ships. """ + arguments = ["build", "--out", str(staging), "--print-manifest"] + if python_version: + arguments += ["--min-version", python_version] + sources: list[str] = [] packages: list[str] = [] - for line in _run_by( - "build", "--out", str(staging), "--print-manifest" - ).splitlines(): + requires: list[str] = [] + for line in _run_by(*arguments).splitlines(): kind, _, value = line.strip().partition(" ") if kind == "input": sources.append(value) elif kind == "package": packages.append(value) - return Staged(sorted(set(sources)), sorted(set(packages))) + elif kind == "requires": + requires.append(value) + return Staged(sorted(set(sources)), sorted(set(packages)), sorted(set(requires))) def _write_staged_pyproject(staging: Path, built: Staged) -> None: @@ -233,6 +244,8 @@ def _write_staged_pyproject(staging: Path, built: Staged) -> None: "to become `app/__init__.by`" ) + metadata["dependencies"] = _merged_dependencies(metadata, built.requires) + document = { "build-system": { "requires": [UV_BUILD_REQUIREMENT], @@ -310,6 +323,34 @@ def _requirements() -> list[str]: # ── the project's own metadata ─────────────────────────────────────────────── +def _merged_dependencies( + metadata: Mapping[str, Any], introduced: Sequence[str] +) -> list[str]: + """The project's dependencies, plus what lowering needs at run time. + + Building for an older python can put a name in the output that only + `typing_extensions` has there, and the project never asked for it, so it + cannot have declared it. A wheel that shipped without it would install + cleanly and fail on the first import of the module that needs it. + + A project that already names the distribution keeps its own constraint: it + knows something about the version it wants that this does not. + """ + declared = list(metadata.get("dependencies", [])) + already = {_requirement_name(requirement) for requirement in declared} + return declared + [ + requirement + for requirement in introduced + if _requirement_name(requirement) not in already + ] + + +def _requirement_name(requirement: str) -> str: + """The distribution a requirement names, normalized.""" + name = re.split(r"[\s<>=!~;\[(]", requirement.strip(), maxsplit=1)[0] + return _normalized_name(name) + + def _read_project_metadata(project_root: Path) -> dict[str, Any]: """The `[project]` table, with anything dynamic settled. diff --git a/scripts/test_build_backend.py b/scripts/test_build_backend.py index cad15370cb..c6260dd12e 100644 --- a/scripts/test_build_backend.py +++ b/scripts/test_build_backend.py @@ -25,6 +25,7 @@ from basedpython.build import ( BuildError, Staged, + _merged_dependencies, _read_project_metadata, _toml, _write_staged_pyproject, @@ -272,3 +273,51 @@ def test_the_staged_tree_is_its_own_module_root(tmp_path: Path) -> None: def test_a_project_with_no_package_to_ship_is_reported(tmp_path: Path) -> None: with pytest.raises(BuildError, match="no package to build a wheel from"): staged_pyproject(tmp_path, PROJECT, Staged(sources=[], packages=[])) + + +# ── what lowering needs at run time ────────────────────────────────────────── + + +def test_a_dependency_lowering_introduced_is_declared() -> None: + """Building for an older python can put a name in the output that only + `typing_extensions` has there. The project never asked for it, so it cannot + have declared it — and a wheel without it fails on the first import.""" + merged = _merged_dependencies( + {"dependencies": ["packaging>=24"]}, ["typing_extensions>=4.12"] + ) + assert merged == ["packaging>=24", "typing_extensions>=4.12"] + + +def test_a_project_with_no_dependencies_still_gets_what_it_needs() -> None: + assert _merged_dependencies({}, ["typing_extensions>=4.12"]) == [ + "typing_extensions>=4.12" + ] + + +def test_nothing_is_added_when_lowering_needed_nothing() -> None: + assert _merged_dependencies({"dependencies": ["packaging>=24"]}, []) == [ + "packaging>=24" + ] + + +def test_a_constraint_the_project_already_declared_is_left_alone() -> None: + """It knows something about the version it wants that this does not.""" + merged = _merged_dependencies( + {"dependencies": ["typing_extensions==4.13.2"]}, ["typing_extensions>=4.12"] + ) + assert merged == ["typing_extensions==4.13.2"] + + +def test_a_declaration_is_matched_however_it_is_spelled() -> None: + """`typing-extensions` and `typing_extensions` are one distribution, and a + requirement can carry an extra, a marker or a comparator after the name.""" + for spelling in ( + "typing-extensions", + "Typing_Extensions >= 4.0", + "typing-extensions[all]>=4", + 'typing_extensions>=4; python_version < "3.11"', + ): + merged = _merged_dependencies( + {"dependencies": [spelling]}, ["typing_extensions>=4.12"] + ) + assert merged == [spelling], spelling From 45466bc1cde665a00adb94585c2c364f79888995 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:09:56 +1000 Subject: [PATCH 3/4] a wheel for each python, rather than one for the oldest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a project builds into one wheel lowered to the oldest python it supports. that wheel runs everywhere, which is the point — but a reader on 3.13 gets code written around 3.9's limits and a `typing_extensions` dependency they have no use for. `python-version` as a build config setting lowers a wheel to one python and tags it accordingly, so an installer hands every interpreter the best wheel it can use and a python with no wheel of its own falls back to the newest one below it. the tag is rewritten after the fact — in `WHEEL`, in the `RECORD` line carrying that file's hash, and in the name — because the backend that builds it has none. `by build --wheels` drives `uv` once per version. it is not a loop for its own sake: a release is only useful if the set is complete, so nothing reaches `dist/` unless every version built, a version with no wheel is refused, and an untagged wheel left beside the others is refused outright — an installer would prefer it to every one of them, and none would ever be chosen. the versions come from `requires-python`, so most projects configure nothing. --- crates/ty/docs/cli.md | 8 +- crates/ty/docs/configuration.md | 36 ++ crates/ty/src/args.rs | 15 +- crates/ty/src/by_wheels.rs | 464 ++++++++++++++++++++++ crates/ty/src/lib.rs | 16 +- crates/ty/tests/by_e2e.rs | 57 +++ crates/ty_project/src/metadata/options.rs | 21 + docs/basedpython/packaging.md | 45 +++ python/basedpython/build.py | 191 ++++++++- scripts/test_build_backend.py | 151 +++++++ ty.schema.json | 11 + 11 files changed, 1002 insertions(+), 13 deletions(-) create mode 100644 crates/ty/src/by_wheels.rs diff --git a/crates/ty/docs/cli.md b/crates/ty/docs/cli.md index 75d53e7d98..1fbd5838b7 100644 --- a/crates/ty/docs/cli.md +++ b/crates/ty/docs/cli.md @@ -297,12 +297,14 @@ by build [OPTIONS]
--help, -h

Print help (see a summary with '-h')

--min-version version

minimum Python version the output must run on [default: the project's configured python version]

--no-unique-loop-bindings

leave a closure made inside a loop sharing the loop's one binding, as python does, instead of binding the values of the iteration it was made in

-
--out, -o dir

Where to write the built project

-

[default: out]

--print-manifest

Report what the build read and produced, as <kind> <value> lines.

+
--out, -o dir

Where to write the output [default: out, or dist with --wheels]

+
--print-manifest

Report what the build read and produced, as <kind> <value> lines.

input <path> for every file the project is made of — what a source distribution has to carry to rebuild into the same thing — and package <name> for every top-level package that came out.

--runtime-raises-checks

wrap every function with a raises clause in a runtime guard that fails when it raises something the clause does not include

--soundness spec

which runtime type-soundness checks to insert: default, all (adds the opt-in parameters entry checks), none, or a comma-separated subset of generic-calls, projections, iterations, assignments, returns, arguments, parameters

-

[default: default]

+

[default: default]

--wheels

Build one publishable wheel per python version, and a source distribution, into dist/.

+

Each wheel is lowered to the version it is tagged for, so an installer hands every interpreter the best wheel it can use rather than one lowered to the oldest python the project supports. Needs uv, which does the packaging.

+
## by compile diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index 0902a8bc7a..6884915c55 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -994,6 +994,42 @@ The value is a path relative to the project root. --- +### `wheel-versions` + +The python versions to build a wheel for, one wheel each. + +`by build --wheels` builds one wheel per version listed and tags each for +the python it was lowered to, so an installer hands every interpreter the +best wheel it can use. A python with no wheel of its own takes the newest +one below it. + +Defaults to every version from the one the project targets up to the +newest this release knows about — which is what `requires-python` already +says the project supports, so most projects need not set this. List them +explicitly to ship fewer. + +**Default value**: `null` + +**Type**: `list[str]` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.build] + wheel-versions = ["3.9", "3.13"] + ``` + +=== "ty.toml" + + ```toml + [build] + wheel-versions = ["3.9", "3.13"] + ``` + +--- + ## `editor` ### `common-aliases` diff --git a/crates/ty/src/args.rs b/crates/ty/src/args.rs index 1ac92d09c3..0063da1721 100644 --- a/crates/ty/src/args.rs +++ b/crates/ty/src/args.rs @@ -137,9 +137,18 @@ pub(crate) enum Command { /// [default: the project's configured python version] #[arg(long, value_name = "VERSION")] min_version: Option, - /// Where to write the built project. - #[arg(short = 'o', long, value_name = "DIR", default_value = "out")] - out: PathBuf, + /// Build one publishable wheel per python version, and a source + /// distribution, into `dist/`. + /// + /// Each wheel is lowered to the version it is tagged for, so an + /// installer hands every interpreter the best wheel it can use rather + /// than one lowered to the oldest python the project supports. Needs + /// `uv`, which does the packaging. + #[arg(long, conflicts_with_all = ["min_version", "print_manifest"])] + wheels: bool, + /// Where to write the output [default: `out`, or `dist` with `--wheels`] + #[arg(short = 'o', long, value_name = "DIR")] + out: Option, /// Report what the build read and produced, as ` ` lines. /// /// `input ` for every file the project is made of — what a source diff --git a/crates/ty/src/by_wheels.rs b/crates/ty/src/by_wheels.rs new file mode 100644 index 0000000000..7ace66e683 --- /dev/null +++ b/crates/ty/src/by_wheels.rs @@ -0,0 +1,464 @@ +//! Building a release: one wheel per python, and the source distribution. +//! +//! A project builds into one wheel by default, lowered to the oldest python it +//! supports. That wheel runs everywhere, which is the point — but it means a +//! reader on 3.13 gets code written around 3.9's limits, and a `typing_extensions` +//! dependency they have no use for. Lowering each wheel to one python and tagging +//! it accordingly lets an installer hand every interpreter the best wheel it can +//! use, and a python with no wheel of its own falls back to the newest one below +//! it. +//! +//! Nothing here packages anything. `uv` is the build frontend, called once per +//! version exactly as it would be from a shell; what this adds is the part a +//! shell loop gets wrong. A release is only useful if the set is *complete* — +//! every version covered, no untagged wheel left behind to outrank the rest, and +//! nothing published at all if one of them failed. That is the whole reason this +//! is a command rather than three lines of shell. + +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::Context; +use ruff_db::system::{OsSystem, System, SystemPath}; +use ruff_python_ast::PythonVersion as AstPythonVersion; +use ty_project::{Db, ProjectDatabase, ProjectMetadata}; +use ty_site_packages::PythonEnvironment; + +use crate::ExitStatus; + +/// The tag every wheel carries when it was not lowered for one python. +/// +/// It is the one that must never appear beside the others: an installer ranks it +/// above every `py3X` tag older than the running interpreter, so a single stray +/// generic wheel silently wins over the whole set. +const UNTAGGED_WHEEL_MARKER: &str = "-py3-none-any.whl"; + +#[allow(clippy::print_stderr)] +pub(crate) fn cmd_build_wheels(out: Option<&Path>) -> anyhow::Result { + let cwd = std::env::current_dir().context("failed to get current directory")?; + let destination = cwd.join(out.unwrap_or(Path::new("dist"))); + + let uv = find_uv(&cwd)?; + let versions = wheel_versions(&cwd)?; + if versions.is_empty() { + anyhow::bail!( + "no python versions to build for — `build.wheel-versions` is empty, \ + and a release with no wheels in it is not a release" + ); + } + + eprintln!( + "building for {}", + versions + .iter() + .map(String::as_str) + .collect::>() + .join(", ") + ); + + // nothing reaches the destination until everything succeeded. a half-built + // release published is worse than no release: the versions that did build + // outrank nothing, so an interpreter whose wheel is missing quietly takes an + // older one and no one is told + let staging = tempfile::TempDir::new().context("failed to create temp directory")?; + + run_uv(&uv, &cwd, &["build", "--sdist"], staging.path())?; + for version in &versions { + run_uv( + &uv, + &cwd, + &[ + "build", + "--wheel", + "--config-setting", + &format!("python-version={version}"), + ], + staging.path(), + )?; + } + + let built = verify(staging.path(), &versions)?; + verify_destination(&destination, &built)?; + fs::create_dir_all(&destination) + .with_context(|| format!("could not create {}", destination.display()))?; + for artifact in &built { + let name = artifact + .file_name() + .context("a built artifact has no name")?; + fs::copy(artifact, destination.join(name)) + .with_context(|| format!("could not write {}", destination.join(name).display()))?; + } + + eprintln!(); + for artifact in &built { + if let Some(name) = artifact.file_name().and_then(std::ffi::OsStr::to_str) { + eprintln!("{}", destination.join(name).display()); + } + } + let wheels = built + .iter() + .filter(|artifact| { + artifact + .extension() + .is_some_and(|extension| extension == "whl") + }) + .count(); + eprintln!("\n{wheels} wheel(s) and a source distribution"); + Ok(ExitStatus::Success) +} + +/// Whether this is something a release is made of. +/// +/// The frontend leaves more than artifacts in its output directory — `uv` writes +/// a `.gitignore` — and a release is the wheels and the source distribution, +/// not whatever else is in the folder. +#[expect( + clippy::case_sensitive_file_extension_comparisons, + reason = "the name is lowercased first, and `.tar.gz` is two extensions to `Path`" +)] +fn is_artifact(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(std::ffi::OsStr::to_str) else { + return false; + }; + let name = name.to_ascii_lowercase(); + name.ends_with(".whl") || name.ends_with(".tar.gz") +} + +/// Check that what was built is a release rather than a pile of wheels. +fn verify(staging: &Path, versions: &[String]) -> anyhow::Result> { + let mut artifacts: Vec = fs::read_dir(staging) + .with_context(|| format!("could not read {}", staging.display()))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_file() && is_artifact(path)) + .collect(); + artifacts.sort(); + + let names: BTreeSet = artifacts + .iter() + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .collect(); + + // an untagged wheel outranks every `py3X` tag below the running python, so + // one of these in the set makes every wheel beside it unreachable + if let Some(untagged) = names + .iter() + .find(|name| name.ends_with(UNTAGGED_WHEEL_MARKER)) + { + anyhow::bail!( + "`{untagged}` is not tagged for a python version, and an installer \ + would prefer it over the wheels that are — so none of them would \ + ever be chosen" + ); + } + + for version in versions { + let tag = format!("-py{}-", version.replace('.', "")); + if !names.iter().any(|name| name.contains(&tag)) { + anyhow::bail!( + "nothing was built for python {version}, so an interpreter of that \ + version would fall back to an older wheel without being told" + ); + } + } + + if !names.iter().any(|name| name.ends_with(".tar.gz")) { + anyhow::bail!("no source distribution was built"); + } + + Ok(artifacts) +} + +/// Check that nothing already in the destination will outrank what is about to +/// be put there. +/// +/// This is where the release is published *from*, so it is where a stale artifact +/// does its damage. An untagged wheel of the version being built is the one that +/// matters most — an installer prefers it to every tagged wheel, so the whole set +/// becomes unreachable — but a wheel for a version no longer built is stale in the +/// same way, and both would be uploaded by a `publish` that takes the directory as +/// it finds it. +/// +/// Only this version is examined. An artifact of a *different* version is a +/// previous release, and a resolver picks the version before it picks a tag, so it +/// takes nothing away from this one. +fn verify_destination(destination: &Path, built: &[PathBuf]) -> anyhow::Result<()> { + let Ok(entries) = fs::read_dir(destination) else { + // nothing there yet, which is the common case and nothing to check + return Ok(()); + }; + + let ours: BTreeSet<&str> = built + .iter() + .filter_map(|path| path.file_name()?.to_str()) + .collect(); + let Some(release) = built.iter().find_map(|path| release_prefix(path)) else { + return Ok(()); + }; + + let stale: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_file() && is_artifact(path)) + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .filter(|name| !ours.contains(name.as_str())) + .filter(|name| release_prefix(Path::new(name)).as_deref() == Some(&release)) + .collect(); + + if !stale.is_empty() { + anyhow::bail!( + "`{}` already holds {} of this release that this build did not produce:\n {}\n \ + they would be published alongside it, and an untagged wheel among them \ + outranks every wheel that is tagged — remove them and build again", + destination.display(), + if stale.len() == 1 { + "an artifact" + } else { + "artifacts" + }, + stale.join("\n "), + ); + } + Ok(()) +} + +/// The `name-version` an artifact belongs to, which is what makes two of them +/// part of the same release. +fn release_prefix(path: &Path) -> Option { + let name = path.file_name()?.to_str()?; + if let Some(stem) = name.strip_suffix(".tar.gz") { + return Some(stem.to_owned()); + } + let stem = name.strip_suffix(".whl")?; + // `name-version-python-abi-platform`: everything before the three tag fields + let mut fields: Vec<&str> = stem.split('-').collect(); + if fields.len() < 4 { + return None; + } + fields.truncate(fields.len() - 3); + Some(fields.join("-")) +} + +/// The versions to build for: what the project lists, else every version from +/// the one it targets up to the newest this release can emit for. +fn wheel_versions(cwd: &Path) -> anyhow::Result> { + let Some(sys_cwd) = SystemPath::from_std_path(cwd) else { + anyhow::bail!("non-utf8 path: {}", cwd.display()); + }; + let system = OsSystem::new(sys_cwd); + let metadata = ProjectMetadata::discover(sys_cwd, &system) + .with_context(|| format!("failed to discover project at {sys_cwd}"))?; + let db = ProjectDatabase::use_defaults(metadata, system); + + if let Some(listed) = db + .project() + .metadata(&db) + .options() + .build + .as_ref() + .and_then(|build| build.wheel_versions.as_ref()) + { + return Ok(listed.iter().map(|version| (**version).clone()).collect()); + } + + // the floor is what the project already declares it supports, so there is + // nothing else to ask: `requires-python` is what an installer enforces, and + // building below it would ship a wheel no one may install + let floor = db.project().program(&db).python_version(&db); + Ok(AstPythonVersion::iter() + .filter(|version| *version >= floor && *version <= AstPythonVersion::latest()) + .map(|version| version.to_string()) + .collect()) +} + +/// Find the `uv` that will do the packaging. +/// +/// The project environment first, for the same reason `by run` looks there: it +/// is the environment this project is developed in. `PATH` after it. +fn find_uv(cwd: &Path) -> anyhow::Result { + if let Some(sys_cwd) = SystemPath::from_std_path(cwd) { + let system = OsSystem::new(sys_cwd); + if let Ok(Some(environment)) = PythonEnvironment::discover(sys_cwd, &system) { + let binaries = if cfg!(windows) { + environment.sys_prefix().join("Scripts") + } else { + environment.sys_prefix().join("bin") + }; + let candidate = binaries.join(if cfg!(windows) { "uv.exe" } else { "uv" }); + if system.is_file(&candidate) { + return Ok(PathBuf::from(candidate.as_str())); + } + } + } + + which_uv().context( + "could not find `uv`, which is what builds the wheels — \ + `by build --wheels` drives it rather than packaging anything itself. \ + install it, or build a single wheel with `uv build`", + ) +} + +fn which_uv() -> Option { + let name = if cfg!(windows) { "uv.exe" } else { "uv" }; + std::env::var_os("PATH").and_then(|path| { + std::env::split_paths(&path) + .map(|directory| directory.join(name)) + .find(|candidate| candidate.is_file()) + }) +} + +#[allow(clippy::print_stderr)] +fn run_uv(uv: &Path, cwd: &Path, arguments: &[&str], out: &Path) -> anyhow::Result<()> { + let status = Command::new(uv) + .args(arguments) + .arg("--out-dir") + .arg(out) + .current_dir(cwd) + .status() + .with_context(|| format!("could not run `{}`", uv.display()))?; + if !status.success() { + anyhow::bail!( + "`uv {}` failed — nothing was written, because a release missing one \ + of its wheels hands an interpreter an older one without saying so", + arguments.join(" ") + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn staged(names: &[&str]) -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("tempdir"); + for name in names { + fs::write(directory.path().join(name), "").expect("write"); + } + directory + } + + #[test] + fn a_complete_set_verifies() { + let directory = staged(&[ + "thing-1.0.tar.gz", + "thing-1.0-py39-none-any.whl", + "thing-1.0-py310-none-any.whl", + ]); + let built = verify(directory.path(), &["3.9".to_owned(), "3.10".to_owned()]) + .expect("a complete set"); + assert_eq!(built.len(), 3); + } + + /// the failure this command exists to prevent: an untagged wheel outranks + /// every `py3X` tag below the running python, so one of them makes the whole + /// set unreachable + #[test] + fn an_untagged_wheel_beside_the_others_is_refused() { + let directory = staged(&[ + "thing-1.0.tar.gz", + "thing-1.0-py39-none-any.whl", + "thing-1.0-py3-none-any.whl", + ]); + let error = verify(directory.path(), &["3.9".to_owned()]).expect_err("refused"); + assert!(error.to_string().contains("not tagged"), "{error}"); + } + + #[test] + fn a_version_with_no_wheel_is_refused() { + let directory = staged(&["thing-1.0.tar.gz", "thing-1.0-py39-none-any.whl"]); + let error = + verify(directory.path(), &["3.9".to_owned(), "3.13".to_owned()]).expect_err("refused"); + assert!(error.to_string().contains("python 3.13"), "{error}"); + } + + #[test] + fn a_release_without_a_source_distribution_is_refused() { + let directory = staged(&["thing-1.0-py39-none-any.whl"]); + let error = verify(directory.path(), &["3.9".to_owned()]).expect_err("refused"); + assert!(error.to_string().contains("source distribution"), "{error}"); + } + + /// the release an artifact belongs to is what makes two of them the same + /// release, and it is everything before the three tag fields + #[test] + fn an_artifact_names_the_release_it_belongs_to() { + assert_eq!( + release_prefix(Path::new("thing-1.0-py39-none-any.whl")).as_deref(), + Some("thing-1.0") + ); + assert_eq!( + release_prefix(Path::new("thing-1.0.tar.gz")).as_deref(), + Some("thing-1.0") + ); + // a version with its own hyphens still ends where the tags begin + assert_eq!( + release_prefix(Path::new("thing-1.0.dev1-py3-none-any.whl")).as_deref(), + Some("thing-1.0.dev1") + ); + assert_eq!(release_prefix(Path::new("nonsense.whl")), None); + } + + /// the failure the whole command exists to prevent, in the one place it + /// actually happens: a stale untagged wheel left in the directory a release is + /// published from outranks every wheel this build just tagged + #[test] + fn a_stale_untagged_wheel_in_the_destination_is_refused() { + let destination = staged(&["thing-1.0-py3-none-any.whl", "thing-1.0-py39-none-any.whl"]); + let built = vec![ + destination.path().join("thing-1.0-py39-none-any.whl"), + destination.path().join("thing-1.0.tar.gz"), + ]; + let error = verify_destination(destination.path(), &built).expect_err("refused"); + let message = error.to_string(); + assert!(message.contains("thing-1.0-py3-none-any.whl"), "{message}"); + assert!(message.contains("outranks"), "{message}"); + } + + /// a version this build no longer produces is stale in the same way — it would + /// be published, and be chosen by the interpreter it is tagged for + #[test] + fn a_wheel_for_a_version_no_longer_built_is_refused() { + let destination = staged(&["thing-1.0-py38-none-any.whl", "thing-1.0-py39-none-any.whl"]); + let built = vec![destination.path().join("thing-1.0-py39-none-any.whl")]; + let error = verify_destination(destination.path(), &built).expect_err("refused"); + assert!(error.to_string().contains("py38"), "{error}"); + } + + /// a previous release is not a threat to this one: a resolver picks the version + /// before it picks a tag, so an older version's wheels take nothing away + #[test] + fn artifacts_of_another_release_are_left_alone() { + let destination = staged(&["thing-0.9-py3-none-any.whl", "thing-1.0-py39-none-any.whl"]); + let built = vec![destination.path().join("thing-1.0-py39-none-any.whl")]; + verify_destination(destination.path(), &built).expect("another release is not stale"); + } + + #[test] + fn rebuilding_over_this_releases_own_artifacts_is_fine() { + let destination = staged(&["thing-1.0-py39-none-any.whl", "thing-1.0.tar.gz"]); + let built = vec![ + destination.path().join("thing-1.0-py39-none-any.whl"), + destination.path().join("thing-1.0.tar.gz"), + ]; + verify_destination(destination.path(), &built).expect("replacing our own is fine"); + } + + #[test] + fn a_destination_that_does_not_exist_yet_is_fine() { + let directory = tempfile::tempdir().expect("tempdir"); + let absent = directory.path().join("dist"); + verify_destination(&absent, &[absent.join("thing-1.0.tar.gz")]).expect("nothing to check"); + } + + /// `py310` must not be read as `py31`, which is why the tag is matched with + /// its separators rather than as a prefix + #[test] + fn a_version_is_not_matched_by_a_shorter_one() { + let directory = staged(&["thing-1.0.tar.gz", "thing-1.0-py310-none-any.whl"]); + let error = verify(directory.path(), &["3.1".to_owned()]).expect_err("refused"); + assert!(error.to_string().contains("python 3.1"), "{error}"); + } +} diff --git a/crates/ty/src/lib.rs b/crates/ty/src/lib.rs index d34daf8a74..11184bab88 100644 --- a/crates/ty/src/lib.rs +++ b/crates/ty/src/lib.rs @@ -3,6 +3,7 @@ mod by_commands; mod by_init; mod by_source_encoding; mod by_staging; +mod by_wheels; mod logging; mod printer; mod python_version; @@ -10,6 +11,7 @@ mod rule; mod version; use std::io::{BufWriter, Write}; +use std::path::Path; use std::process::{ExitCode, Termination}; use std::sync::{Arc, Mutex}; @@ -138,10 +140,22 @@ fn run_command(command: Command) -> anyhow::Result { } Command::Build { min_version, + wheels, out, print_manifest, lowering, - } => by_commands::cmd_build(min_version.as_deref(), &lowering, &out, print_manifest), + } => { + if wheels { + by_wheels::cmd_build_wheels(out.as_deref()) + } else { + by_commands::cmd_build( + min_version.as_deref(), + &lowering, + out.as_deref().unwrap_or(Path::new("out")), + print_manifest, + ) + } + } Command::Compile { files, output, diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index e9b7618c54..1f74763545 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3321,6 +3321,63 @@ fn build_reports_what_lowering_needs_at_run_time() { ); } +/// the packaging is `uv`'s, so without it there is nothing to drive — and the +/// command has to say that rather than fail somewhere further in +#[test] +fn building_wheels_without_a_frontend_says_what_is_missing() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\nrequires-python = \">=3.12\"\n", + ) + .unwrap(); + let package = dir.path().join("src").join("demo"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--wheels"]) + .current_dir(dir.path()) + // an empty `PATH` is the only way to be sure this machine's `uv` is not + // found, whatever the developer happens to have installed + .env("PATH", "") + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "expected a failure:\n{stderr}"); + assert!( + stderr.contains("could not find `uv`"), + "the message has to name what is missing:\n{stderr}" + ); + assert!( + stderr.contains("uv build"), + "and what to do without it:\n{stderr}" + ); +} + +/// `--wheels` produces a release, `--min-version` produces one tree lowered to +/// one python. asking for both is asking for two different things at once +#[test] +fn building_wheels_refuses_a_single_target_version() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write(dir.path().join("main.by"), "x = 1\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["build", "--wheels", "--min-version", "3.12"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("cannot be used with"), + "clap has to reject the combination:\n{stderr}" + ); +} + // ── running a project, not just its `.by` files ────────────────────────────── /// the same hole at run time, where it is fatal rather than untidy: `by run` diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index 5571505e26..a30068239a 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -1682,6 +1682,27 @@ pub struct BuildOptions { )] pub sources: Option, + /// The python versions to build a wheel for, one wheel each. + /// + /// `by build --wheels` builds one wheel per version listed and tags each for + /// the python it was lowered to, so an installer hands every interpreter the + /// best wheel it can use. A python with no wheel of its own takes the newest + /// one below it. + /// + /// Defaults to every version from the one the project targets up to the + /// newest this release knows about — which is what `requires-python` already + /// says the project supports, so most projects need not set this. List them + /// explicitly to ship fewer. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"null"#, + value_type = r#"list[str]"#, + example = r#" + wheel-versions = ["3.9", "3.13"] + "# + )] + pub wheel_versions: Option>>>, + /// The module to read `__version__` from, when `[project]` declares /// `dynamic = ["version"]`. /// diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index 25ddc67850..c399523db6 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -118,6 +118,51 @@ there takes them back for the build too: exclude = ["!dist"] ``` +## a wheel for each python + +one wheel, lowered to the oldest python the project supports, runs everywhere — +and that is what `uv build` produces. it also means a reader on 3.13 gets code +written around 3.9's limits, and a `typing_extensions` dependency they have no +use for + +`by build --wheels` builds one wheel per version instead, each lowered to the +python it is tagged for: + +```sh +by build --wheels +``` + +```text +building for 3.12, 3.13, 3.14 + +dist/lib9-0.3.0.tar.gz +dist/lib9-0.3.0-py312-none-any.whl +dist/lib9-0.3.0-py313-none-any.whl +dist/lib9-0.3.0-py314-none-any.whl +``` + +an installer picks the best wheel each interpreter can use, and a python with no +wheel of its own takes the newest one below it — so 3.15 takes the 3.14 wheel, +and nothing is left uncovered + +the versions come from `requires-python`, up to the newest this release can emit +for. to ship fewer: + +```toml +[tool.basedpython.build] +wheel-versions = ["3.12", "3.14"] +``` + +`uv` does the packaging, called once per version; `by` runs the loop and checks +the result. nothing reaches `dist/` unless the whole set built, because a release +missing one of its wheels hands that interpreter an older one without saying so + +`dist/` itself is checked too, since that is where a release is published *from*. +an artifact of this release that this build did not produce — an untagged wheel +from an earlier `uv build`, or a version no longer built — is refused, because +`uv publish` takes the directory as it finds it and an untagged wheel outranks +every wheel that is tagged. remove them and build again + ## dependencies lowering adds Building for an older python can put a name in the output that only diff --git a/python/basedpython/build.py b/python/basedpython/build.py index 8cbac2be18..e07d880833 100644 --- a/python/basedpython/build.py +++ b/python/basedpython/build.py @@ -24,6 +24,8 @@ from __future__ import annotations +import base64 +import hashlib import io import os import re @@ -33,6 +35,7 @@ import sysconfig import tarfile import tempfile +import zipfile from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING, Any @@ -46,6 +49,10 @@ # them packages anything UV_BUILD_REQUIREMENT = "uv_build>=0.9,<10" +# the config setting that asks for a wheel lowered to one python rather than to +# the floor of what the project supports +TARGET_VERSION_SETTING = "python-version" + # where `build_editable` stages the project. it is `by build`'s own default # output directory on purpose: an editable install points python at this tree, so # a plain `by build` is what refreshes an editable install @@ -108,8 +115,14 @@ def build_wheel( config_settings: Mapping[str, Any] | None = None, metadata_directory: str | None = None, ) -> str: - with _staged() as staging: - return _delegate("build_wheel", staging, wheel_directory, config_settings) + target = _target_version(config_settings) + with _staged(target) as staging: + built = _delegate("build_wheel", staging, wheel_directory, config_settings) + if target is None: + return built + # a wheel lowered for one python says so, so that a newer interpreter can be + # given a better one. see `_retag` + return _retag(Path(wheel_directory) / built, target) def build_editable( @@ -122,9 +135,15 @@ def build_editable( # makes the install editable at all: transpiled python is what gets imported, # and re-running the build is what updates it staging = Path.cwd() / EDITABLE_STAGING_DIRECTORY - built = _stage(staging) + target = _target_version(config_settings) + built = _stage(staging, target) _write_staged_pyproject(staging, built) - return _delegate("build_editable", staging, wheel_directory, config_settings) + editable = _delegate("build_editable", staging, wheel_directory, config_settings) + if target is None: + return editable + # an editable wheel is a pointer, but it is still selected on its tag, so it + # is tagged for the python it was lowered for like any other + return _retag(Path(wheel_directory) / editable, target) def build_sdist( @@ -174,15 +193,36 @@ def build_sdist( @contextmanager -def _staged() -> Iterator[Path]: +def _staged(python_version: str | None = None) -> Iterator[Path]: """The project, built as python, in a directory that lasts for one hook.""" with tempfile.TemporaryDirectory() as directory: staging = Path(directory) / "build" - built = _stage(staging) + built = _stage(staging, python_version) _write_staged_pyproject(staging, built) yield staging +def _target_version(config_settings: Mapping[str, Any] | None) -> str | None: + """The python this wheel is being lowered for, if one was asked for. + + Without it the build targets what the project declares it supports, which is + the wheel that runs everywhere. With it, the wheel is for one python and is + tagged so that only that python — or the next one up with no wheel of its own + — will choose it. + """ + if not config_settings: + return None + target = config_settings.get(TARGET_VERSION_SETTING) + if target is None: + return None + target = str(target).strip() + if not re.fullmatch(r"3\.\d+", target): + raise BuildError( + f"`{TARGET_VERSION_SETTING}` has to be a python version like `3.12`, not `{target}`" + ) + return target + + class Staged: """What a build read, what it produced, and what the result needs to run.""" @@ -266,6 +306,142 @@ def _write_staged_pyproject(staging: Path, built: Staged) -> None: (staging / "pyproject.toml").write_text(_toml(document), encoding="utf-8") +# ── tagging a wheel for the python it was lowered for ──────────────────────── + + +def _retag(wheel: Path, python_version: str) -> str: + """Re-tag `wheel` for one python, and return its new name. + + A wheel's tag is what an installer selects on. Left as `py3-none-any`, every + wheel of a release looks equally good and the first one found wins — so a + 3.13 user could be handed code lowered for 3.9, with a `typing_extensions` + dependency they have no use for. Tagged `py313-none-any`, that same user gets + the wheel built for them, and a python with no wheel of its own falls back to + the newest one below it. + + The backend that built it has no option for this, so the tag is rewritten + here: in the archive's `WHEEL`, in the `RECORD` line that carries that file's + hash, and in the file name itself. + """ + tag = _python_tag(python_version) + entries = _read_wheel(wheel) + + wheel_metadata = _dist_info_entry(entries, "WHEEL") + metadata_info, metadata = entries[wheel_metadata] + entries[wheel_metadata] = (metadata_info, _replace_tag(metadata, tag)) + + record = _dist_info_entry(entries, "RECORD") + record_info, recorded = entries[record] + entries[record] = ( + record_info, + _rerecord(recorded, wheel_metadata, entries[wheel_metadata][1]), + ) + + renamed = wheel.with_name(_retagged_name(wheel.name, tag)) + _write_wheel(renamed, entries) + if renamed != wheel: + wheel.unlink() + return renamed.name + + +def _python_tag(python_version: str) -> str: + return "py" + python_version.replace(".", "") + + +def _retagged_name(name: str, tag: str) -> str: + """`thing-1.0-py3-none-any.whl` under a new python tag. + + The three tag fields are the last three before the extension, whatever the + name and version in front of them contain. + """ + stem, _, extension = name.rpartition(".") + parts = stem.split("-") + if len(parts) < 4: + raise BuildError(f"`{name}` is not a wheel name this can re-tag") + parts[-3] = tag + return "-".join(parts) + "." + extension + + +def _read_wheel(wheel: Path) -> dict[str, tuple[zipfile.ZipInfo, bytes]]: + with zipfile.ZipFile(wheel) as archive: + return { + info.filename: (info, archive.read(info)) for info in archive.infolist() + } + + +def _write_wheel( + wheel: Path, entries: Mapping[str, tuple[zipfile.ZipInfo, bytes]] +) -> None: + """Write the entries back, each as it arrived but for what changed. + + The original `ZipInfo` is reused rather than rebuilt, because it carries what + a wheel means by it: the mode, and so whether a file in `.data/scripts/` is + still executable once installed. Building a fresh one silently made every + entry `0o644`. + """ + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for info, payload in entries.values(): + archive.writestr(info, payload) + + +def _dist_info_entry( + entries: Mapping[str, tuple[zipfile.ZipInfo, bytes]], name: str +) -> str: + matches = [ + entry + for entry in entries + if entry.endswith(f".dist-info/{name}") and entry.count("/") == 1 + ] + if len(matches) != 1: + raise BuildError(f"the wheel does not hold exactly one `{name}` to rewrite") + return matches[0] + + +def _replace_tag(wheel_metadata: bytes, tag: str) -> bytes: + """Rewrite every `Tag:` line to name `tag`. + + A pure-python wheel carries one; rewriting all of them means a wheel that + somehow carried several is either re-tagged wholly or not at all. + """ + lines = wheel_metadata.decode("utf-8").splitlines(keepends=True) + rewritten = [] + seen = False + for line in lines: + if line.startswith("Tag:"): + seen = True + _, _, rest = line.partition(":") + fields = rest.strip().split("-") + fields[0] = tag + rewritten.append("Tag: " + "-".join(fields) + "\n") + else: + rewritten.append(line) + if not seen: + raise BuildError("the wheel's `WHEEL` has no `Tag:` to rewrite") + return "".join(rewritten).encode("utf-8") + + +def _rerecord(record: bytes, path: str, payload: bytes) -> bytes: + """Restate `path`'s hash and size in `RECORD`. + + `RECORD` is what an installer verifies the archive against, so a file + rewritten without it is a wheel that reports itself as corrupt. + """ + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=") + line = f"{path},sha256={digest.decode('ascii')},{len(payload)}\n" + + rewritten = [] + seen = False + for existing in record.decode("utf-8").splitlines(keepends=True): + if existing.split(",")[0] == path: + seen = True + rewritten.append(line) + else: + rewritten.append(existing) + if not seen: + raise BuildError(f"`RECORD` does not mention `{path}`") + return "".join(rewritten).encode("utf-8") + + # ── delegation ─────────────────────────────────────────────────────────────── @@ -285,6 +461,9 @@ def _delegate( ) from error hook = getattr(uv_build, hook_name) + # the settings are this backend's, not the one it delegates to: `uv_build` + # supports none of its own and warns about every one it is handed + config_settings = None out = os.path.abspath(out_directory) Path(out).mkdir(parents=True, exist_ok=True) previous = Path.cwd() diff --git a/scripts/test_build_backend.py b/scripts/test_build_backend.py index c6260dd12e..2af84cb7ed 100644 --- a/scripts/test_build_backend.py +++ b/scripts/test_build_backend.py @@ -26,7 +26,13 @@ BuildError, Staged, _merged_dependencies, + _python_tag, _read_project_metadata, + _replace_tag, + _rerecord, + _retag, + _retagged_name, + _target_version, _toml, _write_staged_pyproject, ) @@ -321,3 +327,148 @@ def test_a_declaration_is_matched_however_it_is_spelled() -> None: {"dependencies": [spelling]}, ["typing_extensions>=4.12"] ) assert merged == [spelling], spelling + + +# ── tagging a wheel for the python it was lowered for ──────────────────────── + + +def test_the_target_version_comes_from_the_config_setting() -> None: + assert _target_version({"python-version": "3.12"}) == "3.12" + assert _target_version(None) is None + assert _target_version({}) is None + assert _target_version({"other": "x"}) is None + + +def test_a_target_that_is_not_a_python_version_is_reported() -> None: + for bad in ("3", "py312", "3.12.1", "latest", ""): + with pytest.raises(BuildError, match="python version"): + _target_version({"python-version": bad}) + + +def test_a_version_becomes_the_tag_an_installer_selects_on() -> None: + assert _python_tag("3.9") == "py39" + assert _python_tag("3.13") == "py313" + + +def test_only_the_python_field_of_the_name_changes() -> None: + assert ( + _retagged_name("thing-1.0-py3-none-any.whl", "py313") + == "thing-1.0-py313-none-any.whl" + ) + # a version with its own hyphens leaves the three tag fields where they are + assert ( + _retagged_name("thing-1.0-rc1-py3-none-any.whl", "py39") + == "thing-1.0-rc1-py39-none-any.whl" + ) + + +def test_a_name_that_is_not_a_wheel_is_reported() -> None: + with pytest.raises(BuildError, match="re-tag"): + _retagged_name("nonsense.whl", "py39") + + +def test_the_tag_line_is_rewritten_and_the_rest_is_left_alone() -> None: + original = b"Wheel-Version: 1.0\nGenerator: uv 0.12.5\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + rewritten = _replace_tag(original, "py311") + assert b"Tag: py311-none-any\n" in rewritten + assert b"Generator: uv 0.12.5\n" in rewritten + assert b"Root-Is-Purelib: true\n" in rewritten + + +def test_a_wheel_with_no_tag_to_rewrite_is_reported() -> None: + with pytest.raises(BuildError, match="`Tag:`"): + _replace_tag(b"Wheel-Version: 1.0\n", "py39") + + +def test_the_record_restates_the_file_that_changed() -> None: + """`RECORD` is what an installer verifies against, so a file rewritten + without it is a wheel that reports itself as corrupt.""" + record = ( + b"thing/__init__.py,sha256=AAA,0\n" + b"thing-1.0.dist-info/WHEEL,sha256=STALE,10\n" + b"thing-1.0.dist-info/RECORD,,\n" + ) + rewritten = _rerecord(record, "thing-1.0.dist-info/WHEEL", b"Tag: py39-none-any\n") + lines = rewritten.decode().splitlines() + assert lines[0] == "thing/__init__.py,sha256=AAA,0" + assert lines[1].startswith("thing-1.0.dist-info/WHEEL,sha256=") + assert not lines[1].endswith("STALE,10") + assert lines[1].endswith(",19") + assert lines[2] == "thing-1.0.dist-info/RECORD,," + + +def test_a_record_that_does_not_mention_the_file_is_reported() -> None: + with pytest.raises(BuildError, match="RECORD"): + _rerecord(b"thing/__init__.py,sha256=AAA,0\n", "missing/WHEEL", b"") + + +def build_wheel_fixture(directory: Path) -> Path: + """A minimal but valid wheel, tagged generically.""" + import zipfile + + wheel = directory / "thing-1.0-py3-none-any.whl" + metadata = b"Wheel-Version: 1.0\nGenerator: test\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("thing/__init__.py", "x = 1\n") + archive.writestr("thing-1.0.dist-info/WHEEL", metadata) + archive.writestr("thing-1.0.dist-info/METADATA", "Name: thing\n") + archive.writestr( + "thing-1.0.dist-info/RECORD", + "thing/__init__.py,sha256=AAA,6\n" + f"thing-1.0.dist-info/WHEEL,sha256=STALE,{len(metadata)}\n" + "thing-1.0.dist-info/RECORD,,\n", + ) + return wheel + + +def test_a_retagged_wheel_keeps_everything_but_its_tag(tmp_path: Path) -> None: + import base64 + import hashlib + import zipfile + + wheel = build_wheel_fixture(tmp_path) + name = _retag(wheel, "3.13") + + assert name == "thing-1.0-py313-none-any.whl" + assert not wheel.exists(), "the wheel under the old name is gone" + + with zipfile.ZipFile(tmp_path / name) as archive: + assert archive.read("thing/__init__.py") == b"x = 1\n" + assert b"Tag: py313-none-any" in archive.read("thing-1.0.dist-info/WHEEL") + + # and the record agrees with what is actually in the archive, or an + # installer reports the wheel as corrupt + payload = archive.read("thing-1.0.dist-info/WHEEL") + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=") + recorded = archive.read("thing-1.0.dist-info/RECORD").decode() + assert f"sha256={digest.decode()},{len(payload)}" in recorded + + +def test_a_retagged_wheel_keeps_the_modes_it_arrived_with(tmp_path: Path) -> None: + """A wheel's entries carry their mode, and an installer honours it — so a + script in `.data/scripts/` that arrives executable has to leave executable. + Rebuilding each entry's metadata instead of reusing it made every one + `0o644`.""" + import zipfile + + wheel = tmp_path / "thing-1.0-py3-none-any.whl" + metadata = b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n" + script = b"#!/bin/sh\necho hi\n" + with zipfile.ZipFile(wheel, "w") as archive: + executable = zipfile.ZipInfo("thing-1.0.data/scripts/tool") + executable.external_attr = (0o755 << 16) | 0o100000 + archive.writestr(executable, script) + archive.writestr("thing-1.0.dist-info/WHEEL", metadata) + archive.writestr( + "thing-1.0.dist-info/RECORD", + f"thing-1.0.data/scripts/tool,sha256=x,{len(script)}\n" + f"thing-1.0.dist-info/WHEEL,sha256=y,{len(metadata)}\n" + "thing-1.0.dist-info/RECORD,,\n", + ) + + name = _retag(wheel, "3.13") + + with zipfile.ZipFile(tmp_path / name) as archive: + mode = archive.getinfo("thing-1.0.data/scripts/tool").external_attr >> 16 + assert mode == 0o755, f"expected 0o755, got {mode:o}" + assert archive.read("thing-1.0.data/scripts/tool") == script diff --git a/ty.schema.json b/ty.schema.json index b17aa6a38b..8256fa45d9 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -306,6 +306,17 @@ "type": "null" } ] + }, + "wheel-versions": { + "description": "The python versions to build a wheel for, one wheel each.\n\n`by build --wheels` builds one wheel per version listed and tags each for\nthe python it was lowered to, so an installer hands every interpreter the\nbest wheel it can use. A python with no wheel of its own takes the newest\none below it.\n\nDefaults to every version from the one the project targets up to the\nnewest this release knows about — which is what `requires-python` already\nsays the project supports, so most projects need not set this. List them\nexplicitly to ship fewer.", + "anyOf": [ + { + "$ref": "#/definitions/Array_of_string" + }, + { + "type": "null" + } + ] } }, "additionalProperties": false From cf088b71cac0d8f5dd7f7d42381bd8ced0997914 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:43:49 +1000 Subject: [PATCH 4/4] `by run` runs on the project's python, from wherever it was called MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit it looked for `.venv` beside the working directory and never read the `environment.python` the project configures — so `by run` in a subdirectory, or in a project whose environment is anywhere but `./.venv`, silently fell through to whatever `python3` `PATH` turned up. `by check` had resolved the environment correctly all along, which is what makes it a discrepancy rather than a gap: two commands disagreeing about what environment this project is. the order now mirrors the checker's — `--python`, then `environment.python`, then an activated venv, conda, or a `.venv` beside the project's `pyproject.toml` — with `$PYTHON` below discovery, since it names an interpreter rather than an environment, and above only the bare `python3` that discovery falls back to. rooting the search at the project rather than the caller turned up the same mistake one layer down: `build_project_db` took the working directory for the project root, so `by run` inside `tests/` transpiled `tests/` alone and could not find the module it was asked to run. --- crates/ty/src/by_commands.rs | 241 ++++++++++++++++++++++-------- crates/ty/tests/by_e2e.rs | 231 ++++++++++++++++++++++++++++ crates/ty_project/src/metadata.rs | 7 +- docs/basedpython/packaging.md | 17 ++- 4 files changed, 430 insertions(+), 66 deletions(-) diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 87937e241b..439343b68d 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -25,34 +25,20 @@ use crate::ExitStatus; use crate::args::LoweringArgs; use crate::by_staging::{Staging, relative_destination, transpiled_destination}; -/// The python version the emitted code must run on when `--min-version` is not -/// given: the one the project configures (`environment.python-version`, else the -/// `requires-python` lower bound), so the checker and the emitter agree about -/// which python this project targets. Falls back to the transpiler's own default -/// outside a project, or when the configuration names no version. -fn configured_min_version(cwd: &Path) -> PythonVersion { - let Some(sys_cwd) = SystemPath::from_std_path(cwd) else { - return Config::default().min_version; - }; - let system = OsSystem::new(sys_cwd); - let Ok(metadata) = ProjectMetadata::discover(sys_cwd, &system) else { - return Config::default().min_version; - }; - let db = ProjectDatabase::use_defaults(metadata, system); - db.project() - .program(&db) - .python_version(&db) - .to_string() - .parse() - .unwrap_or_else(|_| Config::default().min_version) -} - /// The transpile config for a command whose `--min-version` is optional. +/// +/// Without the flag the target is the version the project configures, so that the +/// checker and the emitter agree about which python this project is for. Outside a +/// project — `by transpile` reading a lone file — the transpiler's own default +/// stands in. fn version_config(min_version: Option<&str>, cwd: &Path) -> anyhow::Result { match min_version { Some(spelled) => parse_version(spelled), None => Ok(Config { - min_version: configured_min_version(cwd), + min_version: ResolvedProject::discover(cwd).map_or_else( + |_| Config::default().min_version, + |project| project.python_version(), + ), ..Config::default() }), } @@ -128,7 +114,11 @@ pub(crate) fn cmd_run( python_flag: Option<&Path>, ) -> anyhow::Result { let cwd = std::env::current_dir().context("failed to get current directory")?; - let interpreter = discover_interpreter(python_flag, &cwd); + // one resolution of the project, for the environment and the target version + // both: they are two readings of the same configuration and must not be able + // to disagree + let project = ResolvedProject::discover(&cwd)?; + let interpreter = discover_interpreter(python_flag, &project)?; let python = interpreter.path.clone(); // `run` executes on a specific interpreter, so by default target *its* // version: the emitted code (dataclass `slots=`, PEP 695 syntax, …) must @@ -161,7 +151,7 @@ pub(crate) fn cmd_run( // for (`match`, for one), and the failure lands as a `SyntaxError` inside // generated code rather than as anything the author can act on if let Some(found) = probed { - let configured = configured_min_version(&cwd); + let configured = project.python_version(); if min_version.is_none() && found < configured { anyhow::bail!( "this project targets python {configured}, but the interpreter this would run on \ @@ -577,17 +567,30 @@ fn configured_main(db: &ProjectDatabase) -> Option { /// targeting 3.9 for its whole life without anyone choosing to. pub(crate) fn default_project_python_version() -> PythonVersion { let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - let interpreter = discover_interpreter(None, &cwd); + // `by init` runs before there is a project to resolve, so a failure here is + // the ordinary case rather than a problem + let Ok(project) = ResolvedProject::discover(&cwd) else { + return latest_python_version(); + }; + let Ok(interpreter) = discover_interpreter(None, &project) else { + return latest_python_version(); + }; if interpreter.is_from_path { - return ruff_python_ast::PythonVersion::latest() - .to_string() - .parse() - .unwrap_or_else(|_| Config::default().min_version); + return latest_python_version(); } detect_python_version(&interpreter.path).unwrap_or_else(|| Config::default().min_version) } +/// The newest python this release can emit for. +fn latest_python_version() -> PythonVersion { + ruff_python_ast::PythonVersion::latest() + .to_string() + .parse() + .unwrap_or_else(|_| Config::default().min_version) +} + /// The interpreter `by run` executes on, and how it was chosen. +#[derive(Clone)] struct Interpreter { path: String, origin: String, @@ -600,11 +603,24 @@ struct Interpreter { /// /// The project environment comes first, because that is the environment the /// project *is*: `by check` resolved this project's imports against it, so -/// running against a different python answers a question nobody asked. An -/// explicit choice still wins over discovery — `--python` for this one run, -/// `$PYTHON` for a shell that has already decided — and a bare `python3` off -/// `PATH` is the last resort rather than the first. -fn discover_interpreter(flag: Option<&Path>, root: &Path) -> Interpreter { +/// running against a different python answers a question nobody asked. That has +/// to mean the same environment the checker used, resolved the same way — the +/// `environment.python` the project configures, then an activated virtual +/// environment, a conda environment, or a `.venv` beside the project's +/// `pyproject.toml`. +/// +/// The project root is what all of that is relative to, not the working +/// directory: `by run` from a subdirectory is still this project, and its `.venv` +/// is still the one at the top. +/// +/// `--python` overrides everything, for one run. `$PYTHON` is below discovery +/// because it names an interpreter rather than an environment, so it stands in +/// only where there is no project environment to prefer — but it still beats the +/// bare `python3` that discovery falls back to. +fn discover_interpreter( + flag: Option<&Path>, + project: &ResolvedProject, +) -> anyhow::Result { let named = |path: String, origin: &str| Interpreter { path, origin: origin.to_owned(), @@ -615,47 +631,137 @@ fn discover_interpreter(flag: Option<&Path>, root: &Path) -> Interpreter { // a `--python` may name the interpreter itself or the environment it // lives in, the same way `by check --python` does if flag.is_file() { - return named(flag.display().to_string(), "`--python`"); + return Ok(named(flag.display().to_string(), "`--python`")); } if let Some(interpreter) = interpreter_in_environment(flag, SysPrefixPathOrigin::PythonCliFlag) { - return named(interpreter, "`--python`"); + return Ok(named(interpreter, "`--python`")); } - return named(flag.display().to_string(), "`--python`"); + return Ok(named(flag.display().to_string(), "`--python`")); + } + + // what the project says its environment is, which is what the checker used. + // a configured environment that cannot be resolved is an error rather than + // something to fall past: `by check` refuses it outright, and running on a + // different python than the one just type-checked against — reporting it as + // a version mismatch, which names the wrong cause — is how the two commands + // came to disagree in the first place + if let Some(configured) = project.configured_environment() { + let Some(interpreter) = + interpreter_in_environment(&configured, SysPrefixPathOrigin::PythonCliFlag) + else { + anyhow::bail!( + "`environment.python` is `{}`, which is not a python environment — \ + the same setting `by check` reads, so neither command can use it", + configured.display() + ); + }; + return Ok(named(interpreter, "`environment.python`")); } - if let Ok(python) = std::env::var(EnvVars::PYTHON) { - return named(python, "`PYTHON`"); + let discovered = discovered_environment(project.root()); + if let Some(found) = &discovered + && !found.is_from_path + { + return Ok(found.clone()); } - if let Some(sys_root) = SystemPath::from_std_path(root) { - let system = OsSystem::new(sys_root); - if let Ok(Some(environment)) = PythonEnvironment::discover(sys_root, &system) - && let Some(interpreter) = environment.interpreter(&system) - { - // discovery ends by falling back to whatever python is on `PATH`, - // which is an interpreter but not a *project* environment — the - // difference matters to anything asking what this project targets - let is_from_path = matches!( - environment.origin(), - SysPrefixPathOrigin::PythonBinary | SysPrefixPathOrigin::SelfEnvironment - ); - return Interpreter { - path: interpreter.to_string(), - origin: environment.origin().to_string(), - is_from_path, - }; - } + if let Ok(python) = std::env::var(EnvVars::PYTHON) { + return Ok(named(python, "`PYTHON`")); } - Interpreter { + Ok(discovered.unwrap_or_else(|| Interpreter { path: "python3".to_owned(), origin: "`PATH`".to_owned(), is_from_path: true, + })) +} + +/// A project, resolved once. +/// +/// Discovery walks up from the working directory reading configuration, and +/// every answer taken from it — where the root is, which environment the project +/// declares, which python it targets — has to be the same answer. Resolving it +/// per question was not only repeated work: the copies disagreed about failure, +/// one falling back to the working directory where another gave up. +pub(crate) struct ResolvedProject { + root: PathBuf, + metadata: ProjectMetadata, +} + +impl ResolvedProject { + fn discover(cwd: &Path) -> anyhow::Result { + let sys_cwd = SystemPath::from_std_path(cwd) + .with_context(|| format!("non-utf8 path: {}", cwd.display()))?; + let system = OsSystem::new(sys_cwd); + let metadata = ProjectMetadata::discover(sys_cwd, &system) + .with_context(|| format!("failed to discover project at {sys_cwd}"))?; + let root = PathBuf::from(metadata.root().as_str()); + Ok(Self { root, metadata }) + } + + fn root(&self) -> &Path { + &self.root + } + + /// The environment the project configures, as an absolute path. + fn configured_environment(&self) -> Option { + let sys_root = SystemPath::from_std_path(&self.root)?; + let system = OsSystem::new(sys_root); + let configured = self + .metadata + .options() + .environment + .as_ref()? + .python + .as_ref()?; + Some(PathBuf::from( + configured.absolute(sys_root, &system).as_str(), + )) + } + + /// The python version the emitted code must run on: the one the project + /// configures (`environment.python-version`, else the `requires-python` lower + /// bound), so the checker and the emitter agree about which python this + /// project targets. + fn python_version(&self) -> PythonVersion { + let Some(sys_root) = SystemPath::from_std_path(&self.root) else { + return Config::default().min_version; + }; + let system = OsSystem::new(sys_root); + let db = ProjectDatabase::use_defaults(self.metadata.clone(), system); + db.project() + .program(&db) + .python_version(&db) + .to_string() + .parse() + .unwrap_or_else(|_| Config::default().min_version) } } +/// The environment discovery finds for this project: an activated virtual +/// environment, a conda environment, or a `.venv` at the project root — and, +/// failing all of those, whatever python is on `PATH`. +fn discovered_environment(root: &Path) -> Option { + let sys_root = SystemPath::from_std_path(root)?; + let system = OsSystem::new(sys_root); + let environment = PythonEnvironment::discover(sys_root, &system).ok()??; + let interpreter = environment.interpreter(&system)?; + // discovery ends by falling back to whatever python is on `PATH`, which is + // an interpreter but not a *project* environment — the difference is what + // decides whether it outranks `$PYTHON`, and what a new project targets + let is_from_path = matches!( + environment.origin(), + SysPrefixPathOrigin::PythonBinary | SysPrefixPathOrigin::SelfEnvironment + ); + Some(Interpreter { + path: interpreter.to_string(), + origin: environment.origin().to_string(), + is_from_path, + }) +} + /// The interpreter inside the environment rooted at `path`, if there is one. fn interpreter_in_environment(path: &Path, origin: SysPrefixPathOrigin) -> Option { let sys_path = SystemPath::from_std_path(path)?; @@ -1718,6 +1824,17 @@ fn build_project_db( let system = OsSystem::new(sys_cwd); let project_metadata = ProjectMetadata::discover(sys_cwd, &system) .with_context(|| format!("failed to discover project at {sys_cwd}"))?; + + // the project is the project wherever the command was run from. rooting this + // at the working directory instead means `by run` inside `tests/` transpiles + // `tests/` and nothing else, and then cannot find the module it was asked to + // run — the same mistake as looking for `.venv` beside the caller rather than + // beside the project + let canonical_root = std::fs::canonicalize(project_metadata.root().as_std_path()) + .unwrap_or_else(|_| PathBuf::from(project_metadata.root().as_str())); + let sys_root = SystemPath::from_std_path(&canonical_root) + .with_context(|| format!("non-utf8 path: {}", canonical_root.display()))?; + let metadata = project_metadata.clone(); let db = ProjectDatabase::use_defaults(project_metadata, system); @@ -1741,7 +1858,7 @@ fn build_project_db( // a hidden directory (`.claude/worktrees`, `.venv`, …) holds copies and // dependencies, not this project's sources — emitting them would write // a parallel tree nobody asked for - .filter(|(path, _)| !is_hidden_within(path, &canonical_cwd)) + .filter(|(path, _)| !is_hidden_within(path, &canonical_root)) // nor is the last build's output. it holds a copy of every `.by` source // this build is about to read, and reading those instead would build the // project into itself, one directory deeper each time @@ -1756,10 +1873,10 @@ fn build_project_db( .collect(); let rebuilder = Rebuilder { metadata, - root: sys_cwd.to_path_buf(), + root: sys_root.to_path_buf(), included, }; - Ok((db, sources, rebuilder, canonical_cwd)) + Ok((db, sources, rebuilder, canonical_root)) } /// How much of the check outcome blocks emitting output. diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 1f74763545..f48592be36 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -3496,6 +3496,237 @@ fn running_python_version() -> (u8, u8) { (major, minor) } +/// the project environment is the environment the project *is* — `by check` +/// resolved this project's imports against it, so running against a different +/// python answers a question nobody asked +#[test] +fn run_uses_the_environment_the_project_configures() { + let dir = tempfile::tempdir().expect("tempdir"); + let environment = dir.path().join("environments").join("current"); + let interpreter = fake_environment(&environment); + fs::write( + dir.path().join("pyproject.toml"), + format!( + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.environment]\npython = \"{}\"\n", + "environments/current" + ), + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .env_remove("PYTHON") + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&interpreter), + "expected the configured environment's interpreter ({interpreter}):\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// a `.venv` belongs to the project, not to whichever directory the command was +/// run from — and neither do the sources +#[test] +fn run_from_a_subdirectory_is_still_the_project() { + let dir = tempfile::tempdir().expect("tempdir"); + let environment = dir.path().join(".venv"); + let interpreter = fake_environment(&environment); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.run]\nmain = \"app.main\"\n", + ) + .unwrap(); + let package = dir.path().join("src").join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + fs::write(package.join("main.by"), "print(1)\n").unwrap(); + let elsewhere = dir.path().join("tools"); + fs::create_dir_all(&elsewhere).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("run") + .current_dir(&elsewhere) + .env_remove("PYTHON") + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&interpreter), + "the project's `.venv` is the project's wherever this was run:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn build_from_a_subdirectory_builds_the_project() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + let package = dir.path().join("src").join("app"); + fs::create_dir_all(&package).unwrap(); + fs::write(package.join("__init__.by"), "").unwrap(); + let elsewhere = dir.path().join("tools"); + fs::create_dir_all(&elsewhere).unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .arg("build") + .current_dir(&elsewhere) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "by build failed:\n{stderr}"); + assert!( + elsewhere + .join("out") + .join("app") + .join("__init__.py") + .exists(), + "the module tree is the project's, not the caller's:\n{stderr}" + ); +} + +/// an environment that reports the interpreter it ran, so a test can tell which +/// one `by run` chose. it is a real venv layout, since that is what discovery +/// looks for +fn fake_environment(root: &std::path::Path) -> String { + let binaries = root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + fs::create_dir_all(&binaries).unwrap(); + fs::write(root.join("pyvenv.cfg"), "home = /usr\n").unwrap(); + + let real = which_python(); + let name = if cfg!(windows) { + "python.exe" + } else { + "python3" + }; + let shim = binaries.join(name); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::write( + &shim, + format!( + "#!/bin/sh\necho \"{}\"\nexec {} \"$@\"\n", + shim.display(), + real + ), + ) + .unwrap(); + fs::set_permissions(&shim, fs::Permissions::from_mode(0o755)).unwrap(); + } + #[cfg(not(unix))] + { + fs::copy(&real, &shim).unwrap(); + } + shim.display().to_string() +} + +fn which_python() -> String { + let output = Command::new("python3") + .args(["-c", "import sys; print(sys.executable)"]) + .output() + .expect("python3 is needed to run this test"); + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +/// `$PYTHON` names an interpreter, not an environment, so it stands in only where +/// there is no project environment to prefer. this is a change in what the +/// variable does: it used to be the only mechanism, and so beat everything +#[test] +fn run_prefers_the_project_environment_to_the_python_variable() { + let dir = tempfile::tempdir().expect("tempdir"); + let project = fake_environment(&dir.path().join(".venv")); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n", + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .env("PYTHON", which_python()) + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&project), + "the project's environment outranks `$PYTHON`:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// and where there is no project environment, `$PYTHON` is still what stands in — +/// demoting it below discovery entirely would have made it dead, since discovery +/// always ends at *some* interpreter on `PATH` +#[test] +fn run_falls_back_to_the_python_variable() { + let dir = tempfile::tempdir().expect("tempdir"); + let elsewhere = fake_environment(&dir.path().join("chosen")); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .env("PYTHON", &elsewhere) + .env_remove("VIRTUAL_ENV") + .output() + .expect("failed to spawn by"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&elsewhere), + "with no project environment, `$PYTHON` is the answer:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// a configured environment that cannot be resolved is what `by check` refuses +/// outright. falling past it ran the program on a different python than the one +/// it had just been checked against, and reported that as a version mismatch — +/// naming the wrong cause entirely +#[test] +fn run_refuses_a_configured_environment_that_is_not_one() { + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("pyproject.toml"), + "[project]\nname = \"demo\"\nversion = \"0.1.0\"\n\ + \n[tool.basedpython.environment]\npython = \"absent\"\n", + ) + .unwrap(); + fs::write(dir.path().join("main.by"), "print(1)\n").unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["run", "main"]) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "expected a refusal:\n{stderr}"); + assert!( + stderr.contains("`environment.python`"), + "the message has to name the setting that is wrong:\n{stderr}" + ); +} + /// the shim `by run` puts in the tree it executes is written through the same /// staging as everything else, so a project file of that name is a reported /// collision rather than a silent overwrite diff --git a/crates/ty_project/src/metadata.rs b/crates/ty_project/src/metadata.rs index 4b69f66c13..338ab33006 100644 --- a/crates/ty_project/src/metadata.rs +++ b/crates/ty_project/src/metadata.rs @@ -421,7 +421,12 @@ impl ProjectMetadata { Ok(metadata) } - pub(crate) fn root(&self) -> &SystemPath { + /// The directory the project's configuration was found in. + /// + /// Everything the project declares is relative to this — a configured + /// environment, a source root — so a command run from a subdirectory resolves + /// them against the project rather than against where it happened to be run. + pub fn root(&self) -> &SystemPath { &self.root } diff --git a/docs/basedpython/packaging.md b/docs/basedpython/packaging.md index c399523db6..c6ba8af6a0 100644 --- a/docs/basedpython/packaging.md +++ b/docs/basedpython/packaging.md @@ -238,9 +238,20 @@ it to `app/__init__.by` and it builds ## running on the right python -`by run` uses the project environment: the same interpreter `by check` resolves -imports against, which for a uv project is `.venv`. `$PYTHON` overrides it, and -`by run --python` overrides that +`by run` uses the project environment — the same one `by check` resolves imports +against, resolved the same way: + +1. `by run --python`, for one run +1. the `environment.python` the project configures — and if that names something + that is not an environment, `by run` refuses it, the way `by check` does +1. an activated virtual environment, a conda environment, or a `.venv` beside the + project's `pyproject.toml` +1. `$PYTHON` +1. `python3` on `PATH` + +all of it relative to the project, not to where the command was run: `by run` in a +subdirectory is still this project, uses the project's `.venv`, and builds the +project's modules a project that targets a newer python than the interpreter can run is reported before anything executes: