diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 070026938e..77553c64fa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -305,16 +305,50 @@ jobs: # transitive dep and fails on the api skew. the fork publishes only the # `basedpython` wheel to pypi, never these crates to crates.io - # the project supports 3.13 as well as `PYTHON_VERSION`, and the two differ in what a - # class does with its annotations and in the wording of several exceptions. a job of - # its own rather than a step on `cargo test (linux)`: as a step it pushed that job past - # its budget and the doctest and dogfood steps after it never ran at all - cargo-test-backend-other-python: - name: "cargo test (native backend, python 3.13)" + # `by compile` builds against the running interpreter's headers, links its abi and has + # to answer exactly as that interpreter does, so *which* interpreter is a first-class + # variable and belongs in the workflow rather than in whatever a runner image ships. + # + # the supported set is python 3.13 and 3.14, each in its gil and its free-threaded + # build. `PYTHON_VERSION` is the primary and `cargo test (linux)` runs the whole suite + # against it; this job covers the other three. 3.12 is deliberately absent — it is + # below the project's floor, and it was only ever reached because an image shipped it. + # + # only the version-sensitive targets run here, because only they start an interpreter: + # * the six backend crates, which emit and then load real extension modules + # * `by_transforms`' `*_runtime` tests, which transpile and then execute + # * `ty`'s divergence and end-to-end tests, which do the same + # the rest of the suite reads a vendored typeshed and a configured `python-version`, + # so its answers do not move with the interpreter and repeating it buys nothing. + # + # a job of its own rather than steps on `cargo test (linux)`: as a step it pushed that + # job past its budget and the doctest and dogfood steps after it never ran at all + cargo-test-interpreter: + name: "cargo test (python ${{ matrix.python }})" runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} needs: determine_changes if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} - timeout-minutes: 30 + timeout-minutes: 45 + strategy: + # one interpreter failing says nothing about the others, and which ones are green + # is the whole answer this job exists to give + fail-fast: false + matrix: + include: + - python: "3.13" + backend_filter: "all()" + # the free-threaded builds run the same tests, minus one that cannot be + # observed there: `a_closure_does_not_leak_its_environment` asserts that + # `sys.getrefcount` *rises* while a closure holds a string, and a + # free-threaded interpreter reports the immortal sentinel for that string + # (4294967295 on 3.13t, 3221225472 on 3.14t) whoever holds it. an ordinary + # cpython closure fails the same assertion, so it measures the instrument + # rather than the backend. the leak half of that test — the count is + # unchanged after 20000 closures — runs and passes on both + - python: "3.13t" + backend_filter: "not test(=a_closure_does_not_leak_its_environment)" + - python: "3.14t" + backend_filter: "not test(=a_closure_does_not_leak_its_environment)" env: CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: @@ -338,14 +372,62 @@ jobs: with: version: "0.11.31" enable-cache: "true" + # `PYTHON` is the load-bearing pin: every harness that starts an interpreter reads + # it first. `PATH` is set as well because each of those harnesses falls back to a + # bare name if the variable is ever unset, and `by_transforms` tries `python3.13` + # *before* `python3` — so an unpinned run does not merely drift, it drifts towards + # one particular version. uv's managed installs put a `python3` next to the + # versioned name, and in a free-threaded install that `python3` is itself the + # free-threaded build - name: "Pin the interpreter" + env: + PY: ${{ matrix.python }} + run: | + uv python install "$PY" + python_path="$(uv python find "$PY")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + dirname "$python_path" >> "$GITHUB_PATH" + # a leg that silently resolved to a different interpreter would report a green that + # belongs to some other version, which is the failure mode this whole job is about + - name: "Verify the interpreter is the one asked for" + env: + PY: ${{ matrix.python }} run: | - uv python install 3.13 - echo "PYTHON=$(uv python find 3.13)" >> "$GITHUB_ENV" + echo "PYTHON=$PYTHON" + "$PYTHON" -VV + echo "python3 on PATH: $(command -v python3)" + # a free-threaded leg has to be genuinely free-threaded rather than a gil + # build answering to a `t` name, so the probe reports both facts at once + want_version="${PY%t}" + want="$want_version false" + if [ "$PY" != "$want_version" ]; then + want="$want_version true" + fi + probe='import sys, sysconfig; print("%d.%d" % sys.version_info[:2], "true" if sysconfig.get_config_var("Py_GIL_DISABLED") else "false")' + got=$("$PYTHON" -c "$probe") + echo "PYTHON reports: $got (want: $want)" + [ "$got" = "$want" ] || { echo "::error::PYTHON is not $PY"; exit 1; } + # the harnesses fall back to a bare `python3` when the variable is unset, so + # that has to be the same interpreter too + got_path=$(python3 -c "$probe") + echo "python3 reports: $got_path (want: $want)" + [ "$got_path" = "$want" ] || { echo "::error::python3 on PATH is not $PY"; exit 1; } - name: "Run the native backend" + env: + BACKEND_FILTER: ${{ matrix.backend_filter }} run: | cargo nextest run \ - -p by_build -p by_ir -p by_irbuild -p by_opt -p by_codegen_c -p by_rt + -p by_build -p by_ir -p by_irbuild -p by_opt -p by_codegen_c -p by_rt \ + -E "$BACKEND_FILTER" + # a separate invocation on purpose: the backend's differential tests fork processes + # and compile C, and sharing a run with `by_transforms`' ~1800 other tests starves + # them into failures that pass on their own + - name: "Run the transpiler's runtime tests" + run: cargo nextest run -p by_transforms -E 'binary(/_runtime$/)' + - name: "Run ty's divergence and end-to-end tests" + run: | + cargo nextest run -p ty \ + -E 'binary(mdtest_divergence) + binary(by_e2e) + binary(django_lookup_runtime)' cargo-test-linux: name: "cargo test (linux)" @@ -404,17 +486,24 @@ jobs: # Ignore errors if this step fails; we want to continue to later steps in the workflow anyway. # This step is just to get nice GitHub annotations on the PR diff in the files-changed tab. run: cargo test -p ty_python_semantic --test mdtest || true - # the native backend's tests run against whatever `python3` is first on `PATH`, + # the tests that execute python run against whatever `python3` is first on `PATH`, # which without this is whichever interpreter the runner image happens to ship — # a 3.14 on macos and a 3.12 on linux, so the version under test moved with the # image rather than with the project. `by compile` builds against the running # interpreter's headers and has to answer as that interpreter does, so which one - # it is belongs in the workflow. the other platforms are deliberately left on the - # image's own interpreter: that is coverage on top of this, not instead of it - - name: "Pin the interpreter the native backend is tested against" + # it is belongs in the workflow. + # + # `PATH` as well as `PYTHON`: the harnesses all read the variable first, but each + # falls back to a bare name when it is unset, and `by_transforms` tries + # `python3.13` *before* `python3` — so a run that lost the variable would not + # drift randomly, it would drift to one particular version. the rest of the + # supported set is covered by `cargo test (python …)` + - name: "Pin the interpreter the python-executing tests use" run: | uv python install "${PYTHON_VERSION}" - echo "PYTHON=$(uv python find "${PYTHON_VERSION}")" >> "$GITHUB_ENV" + python_path="$(uv python find "${PYTHON_VERSION}")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + dirname "$python_path" >> "$GITHUB_PATH" - name: "Run tests" run: cargo insta test --all-features --unreferenced reject --test-runner nextest --disable-nextest-doctest - name: "Run doctests" @@ -479,13 +568,30 @@ jobs: - name: "Run doctests" run: cargo test --doc --profile profiling --all-features + # these two used to take whatever interpreter the runner image shipped. that drift did + # find real bugs — a 3.14 macos image surfaced six failures nobody was looking for — + # but it found them as a red that named no version, and a green said nothing about + # which version was covered. the same job could change answer between two reruns of + # the same commit. so the interpreter is pinned here too, and the coverage the drift + # used to buy by accident is bought on purpose by `cargo test (python …)` instead. + # + # the two platforms take different versions, so the platform axis and the version axis + # are both covered without a third job: windows takes the newest because its toolchain + # path is the most version-sensitive code in the backend (it has to name the + # interpreter's import library, whose stem carries both the version and the + # free-threaded `t`), and macos takes the floor version cargo-test-other: strategy: + # each leg is now a distinct platform *and* version, so cancelling the other one + # on the first failure throws away half the answer + fail-fast: false matrix: - platform: - - ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-windows-2022-x86-64-16x32' || 'windows-latest' }} - - ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-macos-15' || 'macos-latest' }} - name: "cargo test (${{ matrix.platform }})" + include: + - platform: ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-windows-2022-x86-64-16x32' || 'windows-latest' }} + python: "3.14" + - platform: ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-macos-15' || 'macos-latest' }} + python: "3.13" + name: "cargo test (${{ matrix.platform }}, python ${{ matrix.python }})" runs-on: ${{ matrix.platform }} needs: determine_changes if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} @@ -519,8 +625,21 @@ jobs: with: version: "0.12.3" enable-cache: "true" + # `PATH` as well as `PYTHON`, for the fallback reason described on + # `cargo test (python …)`. the interpreter reports its own directory rather than + # `dirname` doing it: on windows this shell is git bash but `uv python find` answers + # with a backslash-separated path, which `dirname` reads as a single component + - name: "Pin the interpreter the python-executing tests use" + env: + PY: ${{ matrix.python }} + run: | + uv python install "$PY" + python_path="$(uv python find "$PY")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + "$python_path" -c 'import os, sys; print(os.path.dirname(sys.executable))' >> "$GITHUB_PATH" - name: "Run tests" run: | + python3 -VV cargo nextest run --all-features --profile ci cargo test --all-features --doc @@ -1481,6 +1600,7 @@ jobs: - cargo-fmt - cargo-clippy - cargo-test-linux + - cargo-test-interpreter - cargo-test-wasm - cargo-build-msrv - shellcheck diff --git a/Cargo.lock b/Cargo.lock index ba1fb6ac0e..d0e589bbb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -370,6 +370,7 @@ dependencies = [ "by_ir", "ruff_db", "ruff_python_ast", + "ruff_python_parser", "ruff_python_stdlib", "ruff_text_size", "thin-vec", diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index 8ff93b7e8a..44bbd09001 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -251,6 +251,7 @@ dependencies = [ "by_ir", "ruff_db", "ruff_python_ast", + "ruff_python_parser", "ruff_python_stdlib", "ruff_text_size", "thin-vec", diff --git a/crates/by_build/src/annotate.rs b/crates/by_build/src/annotate.rs index de691cd079..306c4f3bf5 100644 --- a/crates/by_build/src/annotate.rs +++ b/crates/by_build/src/annotate.rs @@ -15,7 +15,7 @@ use by_ir::print::print_function; /// render the report for a lowered module pub fn report(module: &ModuleIr) -> String { - let mut out = format!("# {}\n", module.name); + let mut out = format!("# {}\n", module.name.dotted()); let native: Vec<&Function> = module.all_functions().collect(); let _ = writeln!( diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index ea00a8d7bf..3e47e542b7 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -34,7 +34,7 @@ pub struct Artifact { /// C, and invoke the platform compiler pub fn build_source( source: &str, - module_name: &str, + module_name: impl Into, toolchain: &Toolchain, out_dir: &Path, options: &Options, @@ -87,7 +87,7 @@ pub fn emit_lowered( /// be written, so `--emit-c-only` and a real build report the same layout pub fn emit_source( source: &str, - module_name: &str, + module_name: impl Into, out_dir: &Path, options: &Options, ) -> Result { @@ -109,21 +109,32 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result .with_context(|| format!("could not create {}", out_dir.display()))?; fs::write(out_dir.join(by_rt::BY_H_NAME), by_rt::BY_H)?; - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let source_path = out_dir.join(format!("{last}.c")); + let source_path = out_dir.join(module.name.relative_path(".c")); + create_parent(&source_path)?; fs::write(&source_path, by_codegen_c::emit_module(module)) .with_context(|| format!("could not write {}", source_path.display()))?; Ok(Built { artifact: Artifact { source: source_path, - extension: out_dir.join(format!("{last}.so")), + extension: out_dir.join(module.name.relative_path(".so")), annotation: write_annotation(module, out_dir, options)?, }, declined: module.declined.clone(), }) } +/// make the directory `path` is to be written into +/// +/// an artefact sits at its module's own place in the output tree, so a package +/// member's directory may not exist yet +fn create_parent(path: &Path) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + fs::create_dir_all(parent).with_context(|| format!("could not create {}", parent.display())) +} + /// what a build is allowed to leave interpreted #[derive(Debug, Clone, Default)] pub struct Options { @@ -173,8 +184,8 @@ fn write_annotation( if !options.annotate { return Ok(None); } - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let path = out_dir.join(format!("{last}.annotated")); + let path = out_dir.join(module.name.relative_path(".annotated")); + create_parent(&path)?; fs::write(&path, annotate::report(module)) .with_context(|| format!("could not write {}", path.display()))?; Ok(Some(path)) @@ -191,7 +202,7 @@ fn render_declines<'a>(declines: impl Iterator, options: &Options, version: Option<(u8, u8)>, ) -> Result { @@ -217,9 +228,11 @@ fn finish( // or a debugger lands on source somebody wrote. a caller that knows the real // path sets this itself — the bare module name is the fallback if module.lines.is_none() { - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let path = module + .name + .relative_path(&format!(".{}", options.language.extension())); module.lines = Some(by_ir::function::LineTable::new( - format!("{last}.{}", options.language.extension()), + path.display().to_string(), source, )); } @@ -260,21 +273,25 @@ fn finish( // the version has to be the *interpreter's*, because this python runs inside // the extension at import time — emitting syntax the interpreter cannot parse // makes the whole module fail to load, taking every function with it - if options.language == by_irbuild::Language::Python { - // a `.py` source is already what runs: it is its own fallback - module.fallback_source = Some(source.to_string()); - return Ok(module); - } - let mut config = options.fallback.clone().unwrap_or_default(); - if let Some((major, minor)) = version - && let Ok(parsed) = format!("{major}.{minor}").parse() - { - config.min_version = parsed; - } - let transpiled = by_transforms::transpile(source, &config).map_err(|error| { - anyhow::anyhow!("could not transpile for the interpreted fallback: {error}") - })?; - module.fallback_source = Some(transpiled); + // a `.py` source is already what runs: it is its own fallback + let twin = if options.language == by_irbuild::Language::Python { + source.to_string() + } else { + let mut config = options.fallback.clone().unwrap_or_default(); + if let Some((major, minor)) = version + && let Ok(parsed) = format!("{major}.{minor}").parse() + { + config.min_version = parsed; + } + by_transforms::transpile(source, &config).map_err(|error| { + anyhow::anyhow!("could not transpile for the interpreted fallback: {error}") + })? + }; + // a decorator module init applies to the native definition would otherwise run here + // too, over the twin's — once for each definition rather than once for the name + let twin = by_irbuild::without_init_decorators(&twin, &module) + .map_err(|error| anyhow::anyhow!("could not prepare the interpreted fallback: {error}"))?; + module.fallback_source = Some(twin); Ok(module) } @@ -304,14 +321,17 @@ pub fn build_module(module: &ModuleIr, toolchain: &Toolchain, out_dir: &Path) -> fs::create_dir_all(out_dir) .with_context(|| format!("could not create {}", out_dir.display()))?; + // the header stays at the root of the output tree, and the root is what is put + // on the include path — so one copy serves every artefact however deep its + // package goes let header = out_dir.join(by_rt::BY_H_NAME); let header_changed = write_if_changed(&header, by_rt::BY_H.as_bytes())?; - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let source = out_dir.join(format!("{last}.c")); + let source = out_dir.join(module.name.relative_path(".c")); + create_parent(&source)?; let source_changed = write_if_changed(&source, by_codegen_c::emit_module(module).as_bytes())?; - let extension = out_dir.join(toolchain.extension_file_name(&module.name)); + let extension = out_dir.join(toolchain.extension_path(&module.name)); // the C compiler is by far the slowest step, and the emitted C is a faithful // function of the optimized BIR — so identical C means an identical compile. // keying on the C rather than on the `.by` is what makes a comment-only edit, @@ -470,7 +490,7 @@ mod tests { // returning a float from an int function builder.terminate(Terminator::Return(Value::Float(1.0))); let module = ModuleIr { - name: "bad".to_string(), + name: by_ir::ModuleName::new("bad"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_build/src/toolchain.rs b/crates/by_build/src/toolchain.rs index d5a171c60d..ea51242ed4 100644 --- a/crates/by_build/src/toolchain.rs +++ b/crates/by_build/src/toolchain.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::process::Command; use anyhow::{Context, Result, bail}; +use by_ir::ModuleName; use serde::Deserialize; /// everything needed to compile and link an extension for one interpreter @@ -175,10 +176,10 @@ impl Toolchain { }) } - /// the file name an extension for `module` must have to be importable - pub fn extension_file_name(&self, module: &str) -> String { - let last = module.rsplit('.').next().unwrap_or(module); - format!("{last}{}", self.ext_suffix) + /// where an extension for `module` must sit, relative to the root of an output + /// tree, to be importable under that name + pub fn extension_path(&self, module: &ModuleName) -> PathBuf { + module.relative_path(&self.ext_suffix) } } @@ -234,11 +235,15 @@ mod tests { } #[test] - fn the_extension_name_uses_only_the_last_module_component() { + fn the_extension_path_mirrors_the_module_tree() { let toolchain = Toolchain::from_probe("python3", SAMPLE).unwrap(); assert_eq!( - toolchain.extension_file_name("pkg.app"), - "app.cpython-313-darwin.so" + toolchain.extension_path(&ModuleName::new("pkg.app")), + PathBuf::from("pkg/app.cpython-313-darwin.so") + ); + assert_eq!( + toolchain.extension_path(&ModuleName::package("pkg")), + PathBuf::from("pkg/__init__.cpython-313-darwin.so") ); } diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 4f18de5268..a045a8e33a 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -560,7 +560,7 @@ fn agree_in( language, ..Options::default() }; - let built = match build_source(source, &module, &toolchain, &compiled_dir, &options) { + let built = match build_source(source, module.as_str(), &toolchain, &compiled_dir, &options) { Ok(built) => built, Err(error) => { // only an *absent* toolchain is a skip. anything else is the compiler @@ -2429,743 +2429,1447 @@ def stacked(n: int) -> int: ); } +/// a decorator is evaluated once, and what it did on the way happened once +/// +/// python evaluates it where the `def` stands. the interpreted twin is what stands there +/// and module init evaluates it again over the compiled definition, so `mark` appended +/// twice while the name it left behind was right either way — the whole of the defect was +/// the second append. `mark` returns what it was handed, so nothing but `marked` can +/// show it #[test] -fn string_literals_do_not_leak() { - // `gc.get_objects()` cannot see this: `str` is not GC-tracked, so a leaked - // literal is invisible to an object-count check. the refcount of the returned - // literal is the measurement that works +fn a_decorator_runs_once() { + agree( + "decoratoronce", + "\ +marked: list[int] = [] + + +def mark(f: object) -> object: + marked.append(1) + return f + + +@mark +def counted() -> int: + return 1 + + +@mark +def counted_twice() -> int: + return 2 +", + &["m.counted()", "m.counted_twice()", "m.marked"], + ); +} + +/// a definition the module *reads* keeps its decorator, and declines +/// +/// taking the decorator out of the twin's source leaves the name holding an undecorated +/// definition from the twin's `def` until module init reaches it — a window nothing can +/// see unless the module's own body looks. `AT_IMPORT` looks directly and `alias` keeps +/// what it found, and both of them would otherwise hold what `double` never wrapped +#[test] +fn a_decorated_definition_the_module_reads_declines() { + agree_with_declines( + "decoratorread", + "\ +def double(f) -> object: + def wrapper() -> int: + return f() * 2 + return wrapper + + +@double +def one() -> int: + return 1 + + +AT_IMPORT = one() +alias = one +", + &["m.one()", "m.AT_IMPORT", "m.alias()"], + ); +} + +/// the source both class-decorator tests below compile +/// +/// `mark` hands back what it was given and records only the name, so the binding is right +/// however many times it ran and `seen` is the one thing that can show a second run +const MARKED_CLASSES: &str = "\ +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Marked: + def g(self) -> int: + return 1 + + +@mark +class Second: + def h(self) -> int: + return 2 +"; + +/// a class's decorator is evaluated once, and what it did on the way happened once +/// +/// python runs it where the `class` statement stands. the interpreted twin is what stands +/// there and module init ran it again over the namespace entry the compiled type had +/// taken, so `seen` read `['Marked', 'Second', 'Marked', 'Second']` where python reads +/// `['Marked', 'Second']` — and the class each name ended up bound to was right either +/// way, which is what made it silent +#[test] +fn a_class_decorator_runs_once() { + agree_python( + "classdecoratoronce", + MARKED_CLASSES, + &["m.Marked().g()", "m.Second().h()", "m.seen"], + ); +} + +#[test] +fn a_decorated_class_is_the_compiled_type() { + // the counts above are answered identically by a class that fell back to its + // interpreted definition, so they cannot say which build answered. + // `method_descriptor` can: a compiled type holds one where the interpreted class + // holds a plain function let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_strleak"); + let dir = std::env::temp_dir().join("by_diff_classdecoratoronce_t"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def classify(n: int) -> str: - scratch = \"x\" + \"y\" - if n < 0: - return \"neg\" - return scratch -"; - if build_source( - source, - "by_diff_strleak", + let built = match build_source( + MARKED_CLASSES, + "by_diff_classdecoratoronce_t", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import sys, by_diff_strleak as m\n\ - before = sys.getrefcount(m.classify(-1))\n\ - for _ in range(20000):\n m.classify(-1)\n m.classify(1)\n\ - after = sys.getrefcount(m.classify(-1))\n\ - print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", + "import by_diff_classdecoratoronce_t as m\n\ + print(type(m.Marked.__dict__['g']).__name__,\n\ + \x20 type(m.Second.__dict__['h']).__name__)\n\ + print(m.seen)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n['Marked', 'Second']" ); - assert_eq!(out, "stable"); } +/// a decorated class the module *reads* keeps its decorator, and declines +/// +/// taking the decorator out of the twin's source leaves the interpreted definition +/// standing undecorated from its `class` statement until module init reaches it. `TABLE` +/// looks in that window and keeps what it found, so the list would hold a class the +/// module's own name no longer means — `TABLE[0] is Held` would answer `False` #[test] -fn a_concatenated_operand_keeps_the_callers_reference() { - // an operand handed over to a growing concatenation is a reference the frame - // did not own — the caller's count goes down and the object is freed under it. - // the two builds return the same string either way, so only the count says so +fn a_decorated_class_the_module_reads_declines() { + agree_python_with_declines( + "classdecoratorread", + "\ +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Held: + def value(self) -> int: + return 1 + + +TABLE = [Held] +", + &["m.Held().value()", "m.seen", "m.TABLE[0] is m.Held"], + ); +} + +/// a class named in an *unevaluated* annotation is not read, and keeps compiling +/// +/// `from __future__ import annotations` makes `Held` in that signature a string nothing +/// evaluates, so the module never holds the undecorated definition and the decorator can +/// still move to init. without the future import this is `TABLE = [Held]` again +#[test] +fn a_decorated_class_named_in_a_deferred_annotation_still_compiles() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_strhold"); + let dir = std::env::temp_dir().join("by_diff_classdecoratorannotation"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -def join(a: str, b: str) -> str: - return a + b +from __future__ import annotations -def grow(seed: str, n: int) -> str: - out = seed - i = 0 - while i < n: - out = out + \"x\" - i = i + 1 - return out +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Held: + def value(self) -> int: + return 1 + + +def through(h: Held) -> int: + return h.value() "; - if build_source( + let built = match build_source( source, - "by_diff_strhold", + "by_diff_classdecoratorannotation", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import sys, by_diff_strhold as m\n\ - held = 'a' * 40\n\ - before = sys.getrefcount(held)\n\ - for _ in range(20000):\n m.join(held, 'y')\n m.grow(held, 3)\n\ - after = sys.getrefcount(held)\n\ - print('stable' if after == before else f'moved {before}->{after}')\n", + "import by_diff_classdecoratorannotation as m\n\ + print(type(m.Held.__dict__['value']).__name__, m.through(m.Held()))\n\ + print(m.seen)\n", ); - assert_eq!(out, "stable"); + assert_eq!(out, "method_descriptor 1\n['Held']"); } +/// a decorator written as a chain of attributes, which is what the ir grew an +/// expression to hold +/// +/// the ir carried a single `String` and codegen emitted one interned lookup of it, so +/// `functools.cache` was a name to find whole in the module dict — it was declined +/// rather than compiled, which is why it never raised. every step of a path is a *read*, +/// which is what makes evaluating it at module init mean what it meant where the `def` +/// stood #[test] -fn native_classes_agree() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree( - "classes", +fn a_decorator_written_as_a_path_agrees() { + agree_python( + "pathdeco", "\ -data class Point: - x: int - y: int +import abc +import functools - def total(self) -> int: - return self.x + self.y +CALLS = [] - def scaled(self, k: int) -> int: - return self.total() * k -data class Named: - label: str - count: int +class Wrappers: + @staticmethod + def tag(cls: type) -> type: + cls.tag = 'seen' + return cls - def shout(self) -> str: - return self.label + \"!\" -def make(a: int, b: int) -> object: - return Point(a, b) +@functools.cache +def cached(n: int) -> int: + CALLS.append(n) + return n * 2 + + +class Marks: + @abc.abstractmethod + def area(self) -> int: + return 3 + + +@Wrappers.tag +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +def probe() -> int: + return cached(4) + cached(4) ", &[ - "m.Point(3, 4).total()", - "m.Point(3, 4).scaled(10)", - "[m.Point(a, b).total() for a in (0, -1, 10 ** 20) for b in (0, 5)]", - "m.make(2, 3).total()", - "(m.Point(1, 2).x, m.Point(1, 2).y)", - "m.Named('hi', 1).shout()", - "m.Named('hi', 1).label", - // the argument *count* is checked in both builds - "[(type(e).__name__) for e in [_capture(m.Point, 1)]]", + // the wrapper the decorator returned is what the name holds, and it is + // reached from inside the module as well as outside + "m.probe()", + "m.cached(9)", + // and it ran once, so the second call was a cache hit + "[m.probe(), m.CALLS]", + "type(m.cached).__name__", + // a path off something that is not a module resolves the same way + "m.Held(2).read()", + "m.Held.tag", + // a method's decorator is resolved out of the module namespace too + "m.Marks.area.__isabstractmethod__", + "m.Marks().area()", ], ); } +/// the compiled leg answers the path-decorated definitions +/// +/// the differential legs agree whichever one answered, so a decorator test passes with +/// the codegen path switched off. this is where it is pinned #[test] -fn a_native_constructor_checks_its_argument_types() { - // a documented delta, not an equality: a compiled field is an unboxed - // `ByTagged`, so a `str` cannot be stored there and the check is mandatory. - // `@dataclass` does not enforce annotations at runtime, so the interpreted - // twin accepts it — the same difference `--soundness all` would close +fn a_path_decorated_definition_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_ctorcheck"); + let dir = std::env::temp_dir().join("by_diff_pathdecolive"); let _ = std::fs::remove_dir_all(&dir); - let source = "data class Point:\n x: int\n y: int\n"; - if build_source( + let source = "\ +import abc +import functools + + +class Wrappers: + @staticmethod + def tag(cls: type) -> type: + cls.tag = 'seen' + return cls + + +@functools.cache +def cached(n: int) -> int: + return n * 2 + + +class Marks: + @abc.abstractmethod + def area(self) -> int: + return 3 + + +@Wrappers.tag +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n +"; + let built = match build_source( source, - "by_diff_ctorcheck", + "by_diff_pathdecolive", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - let out = run( + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( &python, &dir, - "import by_diff_ctorcheck as m\n\ - try:\n m.Point('a', 1)\n\ - except TypeError as e:\n print('TypeError:', e)\n\ - else:\n print('accepted a str')\n", + "import by_diff_pathdecolive as m\n\ + print(type(m.cached).__name__, type(m.cached.__wrapped__).__name__)\n\ + print(type(m.Marks.area).__name__, type(m.Held.read).__name__)\n\ + print(m.cached(4), m.Marks.area.__isabstractmethod__, m.Held.tag)\n", + ); + assert_eq!( + out, + "_lru_cache_wrapper builtin_function_or_method\n\ + method method_descriptor\n\ + 8 True seen" ); - assert_eq!(out, "TypeError: expected int, got str"); } +/// a decorator that is a call keeps its decline +/// +/// python calls `mark('x')` where the `def` stands. module-level code is not compiled, +/// so the only moment init has is the end of the module — by which time the interpreted +/// twin has already made that call. making it again would be a second one, in the wrong +/// place, with whatever it did on the way happening twice #[test] -fn a_native_class_has_a_fixed_layout() { +fn a_decorator_that_is_a_call_declines() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_layout"); + let dir = std::env::temp_dir().join("by_diff_calldeco"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int - y: int +MADE = [] + + +def mark(label: str): + MADE.append(label) + + def apply(fn): + fn.label = label + return fn + + return apply + + +@mark('x') +def f(n: int) -> int: + return n + 1 "; - if build_source( + let built = match build_source( source, - "by_diff_layout", + "by_diff_calldeco", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // no `__dict__`, and no attribute outside the declared set + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("run it a second time")), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_layout as m\n\ - p = m.Point(1, 2)\n\ - print(hasattr(p, '__dict__'))\n\ - try:\n p.extra = 1\n\ - except AttributeError:\n print('AttributeError')\n\ - else:\n print('accepted an undeclared attribute')\n", + "import by_diff_calldeco as m\n\ + print(m.f(1), m.f.label, m.MADE, type(m.f).__name__)\n", ); - assert_eq!(out, "False\nAttributeError"); + // the factory ran once, where it was written, and the interpreted definition is + // what the name holds + assert_eq!(out, "2 x ['x'] function"); } +/// a decorator rooted at a name the class body bound keeps its decline +/// +/// `@total.setter` reads the property the body bound above it. a decorator is resolved +/// out of the *module* namespace at init, where there is no such name — the lookup would +/// raise `NameError` and take the whole extension's import with it. that shape also +/// writes two `def`s of one name, which is its own decline and answers first, so +/// `Rooted` stands beside it with the same root and distinct names #[test] -fn a_native_class_instance_does_not_leak() { +fn a_decorator_rooted_in_the_class_body_declines() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_classleak"); + let dir = std::env::temp_dir().join("by_diff_classrooteddeco"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str - n: int -"; - if build_source( - source, - "by_diff_classleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // `tp_dealloc` has to release each refcounted field - let out = run( - &python, - &dir, - "import sys, by_diff_classleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n m.Holder(label, 1)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", - ); - assert_eq!(out, "stable"); -} +class Box: + def __init__(self, n: int) -> None: + self._n = n -#[test] -fn a_declined_function_still_exists_and_behaves_the_same() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 11))) { - return; - } - // the interpreted fallback is what makes coverage total: a construct with no - // native lowering costs speed in that one place and nothing anywhere else - agree_with_declines( - "fallback", - "\ -def fast(a: int) -> int: - return a * 2 + @property + def total(self) -> int: + return self._n -def slow(n: int) -> int: - out = n - try: - pass - except* ValueError: - out = 0 - return out -", - &[ - "m.fast(21)", - "m.slow(21)", - // the declined one is a plain python function in both builds - "type(m.slow).__name__", - ], - ); -} + @total.setter + def total(self, value: int) -> None: + self._n = value -#[test] -fn a_float_module_imports_without_any_extra_runtime_module() { - // `float` transpiles through `JustFloat`, which the lazy-import pass binds - // locally. transpiling with `lazy_imports` off instead emits - // `from ty_extensions import JustFloat`, and the extension then fails to - // import at all — the embedded fallback runs at module init - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_floatimport"); - let _ = std::fs::remove_dir_all(&dir); - let source = "def area(r: float) -> float:\n return 3.0 * r * r\n"; - if build_source( + +class Rooted: + def wrap(fn): + return fn + + @wrap + def value(self) -> int: + return 3 +"; + let built = match build_source( source, - "by_diff_floatimport", + "by_diff_classrooteddeco", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // nothing but the extension itself is on the path + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let reasons = |needle: &str| { + built + .declined + .iter() + .any(|declined| declined.reason.contains(needle)) + }; + assert!( + reasons("`wrap` is bound by the class body"), + "declined: {:?}", + built.declined + ); + assert!( + reasons("`total` is defined more than once"), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_floatimport as m\nprint(m.area(2.0))\n", + "import by_diff_classrooteddeco as m\n\ + b = m.Box(1)\n\ + b.total = 7\n\ + print(b.total, type(m.Box.total).__name__, m.Rooted().value())\n", ); - assert_eq!(out, "12.0"); + assert_eq!(out, "7 property 3"); } -#[test] -fn a_compiled_function_is_a_c_function_object() { - // one of the few places the two builds are *meant* to differ: a natively - // compiled function is a builtin, not a python function. recorded in - // plan.md#semantic-deltas rather than asserted as equal +/// a class whose type slots publish more than its body wrote keeps its decorator's +/// decline +/// +/// python reaches `<=` through `tp_richcompare`, one slot behind all six comparisons — +/// so an emitted type that writes `__lt__` publishes `__le__` as well, answering +/// `NotImplemented`. `functools.total_ordering` reads exactly that: it saw `__le__` +/// already there, filled in nothing, and `a <= b` raised where the interpreted class +/// answered `True`. that was a live wrong answer for the plain-name spelling before the +/// path spelling could reach it at all +#[test] +fn a_class_decorator_over_a_partly_filled_slot_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_cfunc"); + let dir = std::env::temp_dir().join("by_diff_partialslot"); let _ = std::fs::remove_dir_all(&dir); - let source = "def f(a: int) -> int:\n return a\n"; - if build_source( + let source = "\ +import functools + + +@functools.total_ordering +class Ranked: + def __init__(self, n: int) -> None: + self.n = n + + def __eq__(self, other: object) -> bool: + return isinstance(other, Ranked) and self.n == other.n + + def __lt__(self, other: object) -> bool: + return self.n < other.n +"; + let built = match build_source( source, - "by_diff_cfunc", + "by_diff_partialslot", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("publishes `__le__`")), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_cfunc as m\nprint(type(m.f).__name__)\n", + "import by_diff_partialslot as m\n\ + print(m.Ranked(1) <= m.Ranked(2), m.Ranked(3) > m.Ranked(2))\n\ + print(type(m.Ranked.__init__).__name__)\n", ); - assert_eq!(out, "builtin_function_or_method"); + assert_eq!(out, "True True\nfunction"); } #[test] -fn module_level_code_runs_at_import() { - agree( - "modulelevel", +fn a_call_to_a_decorated_function_from_the_same_module_agrees() { + // the module namespace holds what the decorator returned; the native entry holds + // what it was handed. a call written inside the module used to reach the entry, so + // `caller(1)` answered 2 compiled and 4 interpreted with nothing said about it + agree_python( + "decoratedcallee", "\ -LIMIT = 7 +def double(fn): + def inner(x: int) -> int: + return fn(x) * 2 + return inner -def under(n: int) -> int: - return n -def limit() -> int: - return 7 +@double +def f(x: int) -> int: + return x + 1 + + +def caller(x: int) -> int: + return f(x) + + +def plain(x: int) -> int: + return x + 1 + + +def other(x: int) -> int: + return plain(x) ", - &["m.LIMIT", "m.limit()", "m.under(3)"], + &[ + "m.caller(1)", + "m.f(1)", + "m.other(1)", + // the decorator ran exactly once on whatever the name holds, rather than + // once per call site + "[m.caller(1), m.caller(1), m.f(1)]", + ], ); } #[test] -fn no_any_turns_a_gradual_decline_into_an_error() { - let Some((_, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_noany"); - let _ = std::fs::remove_dir_all(&dir); +fn a_construction_of_a_decorated_class_from_the_same_module_agrees() { + // a construction is written against the *name*, and a class decorator is what the + // name then holds. allocating the emitted layout instead skipped the decorator: + // `probe()` answered 3 compiled and 103 interpreted + agree_python( + "decoratedclassctor", + "\ +class Other: + def __init__(self, x: int) -> None: + self.x = x + 100 - let source = "\ -def precise(a: int) -> int: - return a + 1 -def loose(a) -> None: - pass -"; - // by default a gradual parameter compiles — it lands on `object` - let built = build_source( - source, - "by_diff_noany", - &toolchain, - &dir, - &Options::default(), +def swap(cls): + return Other + + +@swap +class C: + def __init__(self, x: int) -> None: + self.x = x + + +def probe(x: int) -> int: + return C(x).x +", + &[ + "m.probe(3)", + "m.C(3).x", + "m.Other(1).x", + // the name is what a construction resolves, from either side of the module + "m.C is m.Other", + ], ); - match built { - Ok(built) => assert!(built.declined.is_empty(), "{:?}", built.declined), - Err(error) => { - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - } +} - // `--no-any` refuses instead, and names the function - let error = build_source( - source, - "by_diff_noany", - &toolchain, - &dir, - &Options { - no_any: true, - ..Options::default() - }, - ) - .unwrap_err() - .to_string(); - assert!(error.contains("no-any"), "{error}"); - assert!(error.contains("loose"), "{error}"); - assert!(error.contains("`a` is gradual"), "{error}"); - // a fully typed function is not blamed - assert!(!error.contains("precise"), "{error}"); +#[test] +fn a_method_modifier_is_not_looked_up_as_a_name() { + // a modifier reaches the ast as a decorator with no `@`. it was emitted as a name + // to look up in the module namespace at init, and there is no such name — so the + // extension built cleanly and then failed to import outright with `NameError: name + // 'override' is not defined`, taking every function in the module with it. + // + // `static` is the sharper case and is not here: it declines, because a method's + // slot zero is forced to the receiver and `staticmethod` says it is not one + agree( + "methodmodifier", + "\ +class Box: + abstract def area(self) -> int: + return 7 + +def probe() -> int: + return Box().area() +", + &[ + "m.probe()", + "m.Box().area()", + // the modifier became `abstractmethod`, so the method carries the same + // marker the interpreted twin carries — not a name nobody bound + "getattr(m.Box.area, '__isabstractmethod__', None)", + ], + ); } #[test] -fn require_native_rejects_any_decline_at_all() { - // a different question from `--no-any`: `list[int]` is not gradual, it is a - // type the compiler does not represent *yet*, so only this flag catches it - let Some((_, toolchain)) = environment() else { +fn a_static_or_class_method_answers_the_same_through_the_class_and_through_an_instance() { + // slot zero used to be forced to the receiver for every method, and these two say + // it is not one — so the compiled `Box.make(3)` bound `3` to a `Box` and raised + // `unbound method Box.make() needs an argument` at its first call. + // + // each is reached four ways: through the class, through an instance, from a method + // of the same class, and from a module-level function. all four go through the + // *descriptor* the type publishes, so a wrong convention shows up in every one + agree_python( + "staticmethod", + "\ +class Box: + @staticmethod + def make(x: int) -> int: + return x + 7 + + @classmethod + def named(cls, y: int) -> str: + return cls.__name__ + str(y) + + def n(self) -> int: + return 1 + + def inside(self) -> int: + return Box.make(1) + len(Box.named(2)) + + +class Alt: + def __init__(self, v: int) -> None: + self.v = v + + @classmethod + def of(cls, v: int) -> \"Alt\": + return cls(v) + + +def probe() -> int: + return Box.make(3) +", + &[ + "m.probe()", + "m.Box.make(3)", + "m.Box().make(3)", + "m.Box.named(2)", + "m.Box().named(2)", + "m.Box().inside()", + "m.Box.make(x=3)", + // a class method is what an alternative constructor is written as, and + // `cls(v)` has to reach the class it was called on + "m.Alt.of(4).v", + "type(m.Alt.of(4)).__name__", + // `__self__` is the class for a class method and nothing at all for a + // static one, on either build + "m.Box.named.__self__ is m.Box", + "getattr(m.Box.make, '__self__', None)", + ], + ); +} + +#[test] +fn a_static_or_class_method_is_the_compiled_one() { + // neither `agree` can say which build answered — a class that declines answers + // identically out of its interpreted definition. `type(C.__dict__['m'])` cannot + // say either: it is `staticmethod` on both legs. + // + // the *descriptor* is what differs. a compiled static or class method is reached + // through a `PyCFunction`, so `type(C.m)` is `builtin_function_or_method` where the + // interpreted leg has a plain `function` or a bound `method` — and the class + // method's dict entry is a `classmethod_descriptor` rather than a `classmethod` + let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_reqnative"); + let dir = std::env::temp_dir().join("by_diff_staticmethod_which"); let _ = std::fs::remove_dir_all(&dir); - // precisely typed and still declines: `except*` has no lowering let source = "\ -def precise(a: int) -> int: - return a + 1 +class Box: + @staticmethod + def make(x: int) -> int: + return x + 7 -def grouped(a: int) -> int: - out = a - try: - pass - except* ValueError: - out = 0 - return out + @classmethod + def named(cls, y: int) -> str: + return cls.__name__ + str(y) "; - // `--no-any` is satisfied: nothing here is gradual - match build_source( + let built = match build_source( source, - "by_diff_reqnative", + "by_diff_staticmethod_which", &toolchain, &dir, &Options { - no_any: true, + language: by_irbuild::Language::Python, ..Options::default() }, ) { - Ok(built) => assert_eq!(built.declined.len(), 1, "{:?}", built.declined), + Ok(built) => built, Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); eprintln!("skipping: no working C toolchain ({error})"); return; } - } - - // `--require-native` is not - let error = build_source( - source, - "by_diff_reqnative", - &toolchain, + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, &dir, - &Options { - require_native: true, - ..Options::default() - }, - ) - .unwrap_err() - .to_string(); - assert!(error.contains("require-native"), "{error}"); - assert!(error.contains("`except*`"), "{error}"); + "import by_diff_staticmethod_which as m\n\ + print(type(m.Box.make).__name__, type(m.Box.__dict__['make']).__name__)\n\ + print(type(m.Box.named).__name__, type(m.Box.__dict__['named']).__name__)\n\ + print(m.Box.make(3), m.Box.named(2))\n", + ); + assert_eq!( + out, + "builtin_function_or_method staticmethod\n\ + builtin_function_or_method classmethod_descriptor\n\ + 10 Box2" + ); } #[test] -fn no_any_accepts_a_fully_typed_module() { - let Some((_, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_noany_ok"); - let _ = std::fs::remove_dir_all(&dir); - let source = "def f(a: int, b: int) -> int:\n return a * b\n"; - match build_source( - source, - "by_diff_noany_ok", - &toolchain, - &dir, - &Options { - no_any: true, - ..Options::default() - }, - ) { - Ok(built) => assert!(built.declined.is_empty()), - Err(error) => eprintln!("skipping: no working C toolchain ({error})"), - } +fn a_static_or_class_method_on_a_class_built_through_its_metaclass_answers() { + // a class on a base out of this module may be built by *calling* its metaclass, + // and that construction puts the method table into a namespace rather than onto a + // type — so the descriptor is one the runtime builds. building the plain kind for + // either of these would hand the function the wrong receiver + agree_python( + "staticmethodmeta", + "\ +import collections.abc + + +class Sized(collections.abc.Sized): + def __len__(self) -> int: + return 3 + + @staticmethod + def tag() -> str: + return \"sized\" + + @classmethod + def kind(cls) -> str: + return cls.__name__ +", + &[ + "m.Sized.tag()", + "m.Sized().tag()", + "m.Sized.kind()", + "m.Sized().kind()", + "len(m.Sized())", + "type(m.Sized).__name__", + ], + ); } #[test] -fn optimized_output_still_agrees_with_the_interpreter() { - // copy propagation and infallibility both rewrite the IR — the differential - // legs are what say the rewrites preserved the program - agree( - "optimized", +fn a_static_method_the_boundary_hands_over_reaches_the_plain_function() { + // a boundary that cannot establish a parameter hands the whole call to the + // interpreted twin, and for a *method* that twin is taken off the class with the + // receiver put back in front of it. a static method has no receiver — `self` holds + // nothing at all — and the twin taken off the class is already the plain function + // the `staticmethod` wraps, so prepending anything would have handed it NULL. + // + // both reasons to hand over are here: `float` admits an `int`, and a default that + // is not an immediate is one object every call has to share + agree_python( + "staticmethoddefer", "\ -def area(r: float) -> float: - scaled = r * r - return 3.5 * scaled +DEFAULT = [1, 2] -def compose(a: float, b: float) -> float: - x = area(a) - y = area(b) - return x + y + +class Box: + @staticmethod + def half(x: float) -> float: + return x / 2 + + @staticmethod + def counted(xs=DEFAULT) -> int: + return len(xs) + + @staticmethod + def nests(n: int) -> int: + def inner(k: int) -> int: + return k + n + return inner(1) + + +def nests(n: int) -> int: + def inner(k: int) -> int: + return k + n + 100 + return inner(1) ", &[ - "m.area(2.0)", - "m.compose(1.0, 2.0)", - "[m.compose(a, a) for a in (0.0, -1.5, 3.25)]", + "m.Box.half(5.0)", + // the int arrives where a `double` was compiled, so this is the call that + // goes back to the interpreted definition + "m.Box.half(5)", + "m.Box().half(5)", + "m.Box.counted()", + "m.Box.counted([1])", + "m.Box.counted() and m.Box.counted() is not None", + // the object the default holds is the module's, shared by every call + "m.Box.counted.__self__ if hasattr(m.Box.counted, '__self__') else None", + // a nested function lives on a generated class named after the frame that + // makes it, and these two frames are both called `nests` + "m.Box.nests(3)", + "m.nests(3)", ], ); } #[test] -fn a_declined_function_is_reported_with_a_reason() { - let Some((_, toolchain)) = environment() else { +fn a_class_method_the_boundary_would_hand_over_declines() { + // the twin a method's boundary hands over to is taken off the interpreted class, + // and for a class method python has already *bound* it — to that class, not to the + // one in slot zero. handing it the class as well would give the body two of them + let Some((python, toolchain)) = environment() else { return; }; - let dir: PathBuf = std::env::temp_dir().join("by_diff_declined"); + let dir = std::env::temp_dir().join("by_diff_classmethod_defer"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def fast(a: int) -> int: - return a + 1 +DEFAULT = [1, 2] -def slow(a: int) -> None: - try: - pass - except* ValueError: - pass + +class Box: + @classmethod + def counted(cls, xs=DEFAULT) -> int: + return len(xs) "; - let Ok(built) = build_source( + let built = match build_source( source, - "by_diff_declined", + "by_diff_classmethod_defer", &toolchain, &dir, - &Options::default(), - ) else { - eprintln!("skipping: no working C toolchain"); - return; + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } }; - assert_eq!(built.declined.len(), 1); - assert_eq!(built.declined[0].name, "slow"); assert!( - built.declined[0].reason.contains("`except*`"), - "{:?}", - built.declined[0] + built.declined.iter().any(|declined| declined + .reason + .contains("already bound to the interpreted class")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_classmethod_defer as m\n\ + print(m.Box.counted(), m.Box.counted([1]))\n", ); + assert_eq!(out, "2 1"); } #[test] -fn field_access_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree( - "fields", +fn a_global_a_frame_assigns_reaches_the_module_namespace() { + // the write half of a `global` declaration. there was no op for it, so the + // assignment bound a local and the module's name kept its old value — a wrong + // answer rather than a missing one, and the reason this shape was declined. + // + // `agree_python` asserts nothing declined, and + // `a_compiled_frame_is_what_reaches_the_module_namespace` below is what says the + // native function is the one python calls + agree_python( + "globalwrite", "\ -data class Point: - x: int - y: int +inited = False +counter = 0 - def total(self) -> int: - return self.x + self.y - def shift(self, d: int) -> int: - self.x = self.x + d - return self.x +def init() -> None: + global inited, counter + inited = True + counter += 1 -data class Line: - a: Point - b: Point - def span(self) -> int: - return self.b.x - self.a.x +def bump(n: int) -> int: + global counter + counter = counter + n + return counter +", + &[ + // read from *outside* after a write from a compiled frame: a register + // write is invisible here, which is the whole of the bug + "(m.inited, m.counter)", + "(m.init(), m.inited, m.counter)", + "(m.init(), m.counter)", + "(m.bump(5), m.counter)", + // and the interpreted world's own write is what the compiled frame's + // next read has to see, since both are the one dict + "(setattr(m, 'counter', 100), m.bump(1), m.counter)", + ], + ); +} - def retarget(self, p: Point) -> int: - self.b = p - return self.span() +#[test] +fn a_global_a_frame_assigns_is_read_back_in_that_same_frame() { + // the other half, and the one a write alone can get wrong in the opposite + // direction: if the write reaches the namespace while a later read in the same + // frame still resolves a register, the two halves stop agreeing with each other. + // + // this is `mimetypes` in miniature — a constructor that initializes the module the + // first time it runs, and an `init` whose flag never landed, so the second + // construction called it again and the compiled leg recursed until the stack ran out + agree_python( + "globalselfread", + "\ +inited = False +log: list[str] = [] -def sum_of(p: Point) -> int: - return p.x + p.y -def bump_twice(p: Point) -> int: - p.shift(1) - p.shift(2) - return p.x +class C: + def __init__(self) -> None: + if not inited: + init() + self.x = 1 + + +def init() -> None: + global inited + log.append('init') + inited = True + # the read the write has to be visible to, in this frame and through `C` + if inited: + C() + + +def flip() -> str: + global inited + inited = not inited + # written, read, written again: three answers out of one place + first = inited + inited = not inited + return f'{first} {inited}' ", &[ - "m.Point(3, 4).total()", - "m.sum_of(m.Point(3, 4))", - "m.Point(3, 4).shift(10)", - "m.bump_twice(m.Point(0, 0))", - "m.Line(m.Point(1, 2), m.Point(10, 20)).span()", - "m.Line(m.Point(1, 2), m.Point(10, 20)).a.y", - "m.Line(m.Point(1, 2), m.Point(10, 20)).retarget(m.Point(100, 0))", - // a write through the python-visible setter, then a native read - "[(p := m.Point(1, 2), setattr(p, 'x', 9), p.total())[-1]]", - "[m.sum_of(m.Point(a, b)) for a in (0, -1, 10 ** 20) for b in (0, 5)]", + "(m.C().x, m.inited, m.log)", + "m.flip()", + "(m.flip(), m.inited)", ], ); } #[test] -fn a_field_setter_checks_its_value() { - // the same documented delta as the constructor: an unboxed field cannot hold - // the wrong representation, and `@dataclass` does not check assignments +fn a_global_a_frame_deletes_leaves_the_name_unbound() { + // `del x` under a `global x` unbinds the module's name, and reading it afterwards + // is a `NameError` — which is not the `KeyError` deleting from a dict raises, nor + // what a register could ever report + agree_python( + "globaldelete", + "\ +value = 1 + + +def drop() -> str: + global value + del value + try: + return repr(value) + except NameError as error: + return f'NameError: {error}' + + +def again() -> str: + global value + try: + del value + except NameError as error: + return f'NameError: {error}' + return 'deleted' + + +def restore(n: int) -> int: + global value + value = n + return value +", + &[ + "m.drop()", + "m.again()", + "(m.restore(7), m.value)", + "(m.drop(), m.again())", + ], + ); +} + +#[test] +fn a_global_a_nested_frame_declares_is_not_the_enclosing_local_of_that_name() { + // the enclosing frame binds a local `seen` and the nested one declares `seen` + // global, so they are two different places. deciding captures without consulting + // the declaration makes the closure read and write the enclosing local instead, + // and both the module's name and the local then answer wrongly + agree_python( + "globalnested", + "\ +seen = 0 +tally = 0 + + +def outer(n: int) -> str: + seen = n + + def inner() -> int: + global seen + seen = 5 + return seen + + return f'{inner()} {seen}' + + +def only_reads(n: int) -> str: + # the nested frame declares the name and never writes it, so nothing about *it* + # says the enclosing local is the wrong place — only the declaration does + seen = n + + def peek() -> int: + global seen + return seen + + return f'{peek()} {seen}' + + +def declared_out_here(n: int) -> str: + # and the mirror: the enclosing frame declares it and writes it, so it has no + # register for the nested frame to capture even though the name looks local + global tally + tally = n + + def peek() -> int: + return tally + + return f'{peek()} {tally}' + + +def shadow(n: int) -> int: + # no declaration: an ordinary local that shadows the module's name + seen = n + return seen +", + &[ + "(m.outer(3), m.seen)", + "(m.only_reads(9), m.seen)", + "(m.declared_out_here(4), m.tally)", + "(m.shadow(9), m.seen)", + ], + ); +} + +#[test] +fn a_global_a_generator_assigns_is_not_one_of_its_state_fields() { + // a generator's locals become fields of the state object, because the frame has to + // survive a suspension. a declared `global` is not one of them — the module + // namespace already outlives every suspension — so it must be kept out of that + // layout as much as out of a register, and each resumption has to write through + agree_python( + "globalgen", + "\ +total = 0 +steps: list[int] = [] + + +def counting(n: int): + global total + for i in range(n): + total = total + i + # the write has to be visible across the suspension, from outside and back + yield total + + +def resets(): + global total + total = 0 + yield total + total = 100 + yield total +", + &[ + "(list(m.counting(4)), m.total)", + "(list(m.counting(3)), m.total)", + // stepped by hand, reading the module's name between resumptions + "[(next(g), m.total) for g in [m.counting(5)] for _ in range(3)]", + "(list(m.resets()), m.total)", + ], + ); +} + +#[test] +fn a_module_level_name_a_global_rebinds_is_found_through_the_namespace() { + // taking the write opened this: a frame can now rebind a name the module *defined*, + // and a call written against that name was reaching the definition directly. so the + // rebound function went on answering with the old body — the same thing a decorator + // does to a name, and it goes in the same set + agree_python( + "globalrebind", + "\ +def base() -> int: + return 1 + + +def other() -> int: + return 99 + + +def calls_base() -> int: + return base() + + +def rebind() -> str: + global base + base = other + return f'{base()} {calls_base()}' + + +def replaces_itself() -> int: + # `pydoc.pager` is this exactly: decide once what to be, rebind the name, then + # call *through the name*. reaching the native entry for that last call re-enters + # the body that just rebound it, and the stack runs out + global replaces_itself + replaces_itself = other + return replaces_itself() +", + &[ + "(m.calls_base(), m.base())", + "m.rebind()", + "(m.calls_base(), m.base())", + "m.replaces_itself()", + "(m.replaces_itself(), m.replaces_itself())", + ], + ); +} + +#[test] +fn a_compiled_frame_is_what_reaches_the_module_namespace() { + // the differential tests above compare two legs, and a leg that fell back to its + // interpreted definition answers exactly as the interpreted leg does — so they + // cannot say *which* build wrote the global. this one can: a module-level function + // python calls through `PyModule_AddFunctions` is a `builtin_function_or_method`, + // and one that fell back is a `function` let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_setcheck"); + let dir = std::env::temp_dir().join("by_diff_globalidentity"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int - y: int +inited = False -frozen data class Fixed: - n: int + +class C: + def __init__(self) -> None: + if not inited: + init() + self.x = 1 + + +def init() -> None: + global inited + inited = True + C() "; - if build_source( + let built = match build_source( source, - "by_diff_setcheck", + "by_diff_globalidentity", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); let out = run( &python, &dir, - "import by_diff_setcheck as m\n\ - p = m.Point(1, 2)\n\ - p.x = 5\n\ - print(p.x)\n\ - for attempt in (lambda: setattr(p, 'x', 'a'), lambda: delattr(p, 'x'), lambda: setattr(m.Fixed(1), 'n', 2)):\n\ - \x20 try:\n attempt()\n\ - \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ - \x20 else:\n print('accepted')\n", + "import by_diff_globalidentity as m\n\ + print(type(m.init).__name__, m.C().x, m.inited)\n", ); - assert_eq!(out, "5\nTypeError\nAttributeError\nAttributeError"); + // and `m.inited` read from out here is the module's own binding, which a register + // write never touched. before there was an op for it, `C()` inside `init` saw the + // old `False` and called `init` again until the stack ran out + assert_eq!(out, "builtin_function_or_method 1 True"); } #[test] -fn a_class_typed_argument_is_checked_at_the_boundary() { - // a compiled parameter typed as a native class is a pointer to its struct, - // so python handing over anything else has to be caught here — the - // alternative is a wild pointer dereference +fn a_declined_function_reads_the_global_a_compiled_one_wrote() { + // the asymmetry the whole thing turns on. a compiled frame and the interpreted + // twin of a *declined* one are the same module, and the twin's `__globals__` is + // the dict the compiled frame binds into — so a write is visible to it at once. + // a register write is visible to nobody, which is why this had to decline let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_argcheck"); + let dir = std::env::temp_dir().join("by_diff_globaltwin"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int +flag = 0 -def read(p: Point) -> int: - return p.x + +def writes(n: int) -> int: + global flag + flag = n + return flag + + +def declines_and_reads() -> str: + # `del` on a plain local has no lowering, so this whole function stays interpreted + tmp = 1 + del tmp + return f'{flag}' "; - if build_source( + let built = match build_source( source, - "by_diff_argcheck", + "by_diff_globaltwin", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // the two legs of the same module: one compiled, one not. if `del` on a local ever + // gains a lowering this stops being true, and the assertion says so rather than + // quietly testing two compiled functions + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + vec!["declines_and_reads"], + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_argcheck as m\n\ - print(m.read(m.Point(7)))\n\ - for bad in ('a', 1, None):\n\ - \x20 try:\n m.read(bad)\n\ - \x20 except TypeError as e:\n print(e)\n\ - \x20 else:\n print('accepted')\n", - ); - assert_eq!( - out, - "7\nexpected by_diff_argcheck.Point, got str\n\ - expected by_diff_argcheck.Point, got int\n\ - expected by_diff_argcheck.Point, got NoneType" + "import by_diff_globaltwin as m\n\ + print(type(m.writes).__name__, type(m.declines_and_reads).__name__,\n\ + \x20 m.declines_and_reads(), m.writes(42), m.declines_and_reads(), m.flag)\n", ); + assert_eq!(out, "builtin_function_or_method function 0 42 42 42"); } #[test] -fn a_field_read_does_not_leak() { +fn a_second_decorator_over_a_static_method_declines() { + // the runtime folds the rest of a method's decorators onto the attribute it reads + // back off the finished type — and reading a static method back hands over the + // plain function it wraps, which would be written back as an ordinary method. so + // the pair keeps the decline, and the interpreted definition is what answers let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_fieldleak"); + let dir = std::env::temp_dir().join("by_diff_staticmethod_stacked"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str +def mark(fn): + fn.marked = True + return fn -def label_of(h: Holder) -> str: - return h.label -data class Nest: - inner: Holder +class Stacked: + @mark + @staticmethod + def both() -> int: + return 1 +"; + let built = match build_source( + source, + "by_diff_staticmethod_stacked", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("a second decorator over")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_staticmethod_stacked as m\n\ + print(m.Stacked.both(), type(m.Stacked.__dict__['both']).__name__)\n", + ); + assert_eq!(out, "1 staticmethod"); +} -def inner_label(n: Nest) -> str: - return n.inner.label +#[test] +fn string_literals_do_not_leak() { + // `gc.get_objects()` cannot see this: `str` is not GC-tracked, so a leaked + // literal is invisible to an object-count check. the refcount of the returned + // literal is the measurement that works + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_strleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def classify(n: int) -> str: + scratch = \"x\" + \"y\" + if n < 0: + return \"neg\" + return scratch "; if build_source( source, - "by_diff_fieldleak", + "by_diff_strleak", &toolchain, &dir, &Options::default(), @@ -3175,117 +3879,73 @@ def inner_label(n: Nest) -> str: eprintln!("skipping: no working C toolchain"); return; } - // a field read hands back a *borrowed* reference, so the register that takes - // it must retain — and release. `str` is not gc-tracked, so this measures the - // refcount rather than the object count let out = run( &python, &dir, - "import sys, by_diff_fieldleak as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - n = m.Nest(h)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.label_of(h)\n\ - \x20 m.inner_label(n)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n", + "import sys, by_diff_strleak as m\n\ + before = sys.getrefcount(m.classify(-1))\n\ + for _ in range(20000):\n m.classify(-1)\n m.classify(1)\n\ + after = sys.getrefcount(m.classify(-1))\n\ + print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", ); assert_eq!(out, "stable"); } #[test] -fn a_constructor_result_is_used_natively() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); +fn a_concatenated_operand_keeps_the_callers_reference() { + // an operand handed over to a growing concatenation is a reference the frame + // did not own — the caller's count goes down and the object is freed under it. + // the two builds return the same string either way, so only the count says so + let Some((python, toolchain)) = environment() else { return; - } - agree( - "ctornative", - "\ -data class Point: - x: int - y: int - - def total(self) -> int: - return self.x + self.y - -data class Line: - a: Point - b: Point - -def diag(a: int) -> int: - return Point(a, a).x - -def build(a: int, b: int) -> int: - return Line(Point(a, b), Point(b, a)).b.x - -def widened(a: int) -> object: - return Point(a, a) - -def chained(a: int) -> int: - p = Point(a, a + 1) - q = Point(p.y, p.x) - return q.total() + p.total() -", - &[ - "m.diag(7)", - "m.build(3, 9)", - "m.widened(4).total()", - "m.chained(5)", - "[m.chained(a) for a in (0, -3, 10 ** 20)]", - ], - ); -} + }; + let dir = std::env::temp_dir().join("by_diff_strhold"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def join(a: str, b: str) -> str: + return a + b -#[test] -fn a_declined_callee_does_not_break_the_build() { - // the whole module used to fail to compile: the emitted call named a symbol - // that was never defined - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); +def grow(seed: str, n: int) -> str: + out = seed + i = 0 + while i < n: + out = out + \"x\" + i = i + 1 + return out +"; + if build_source( + source, + "by_diff_strhold", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); return; } - agree_with_declines( - "declinechain", - "\ -def helper[T](a: T) -> T: - return a - -def caller(a: int) -> int: - return helper(a) + a - -data class Point: - x: int - - def bad[T](self, a: T) -> int: - return self.x - -def read(p: Point) -> int: - return p.x - -def alone(a: int) -> int: - return a + 1 -", - &[ - "m.helper(1)", - "m.caller(2)", - "m.read(m.Point(5))", - "m.Point(5).bad(1)", - "m.alone(3)", - ], + let out = run( + &python, + &dir, + "import sys, by_diff_strhold as m\n\ + held = 'a' * 40\n\ + before = sys.getrefcount(held)\n\ + for _ in range(20000):\n m.join(held, 'y')\n m.grow(held, 3)\n\ + after = sys.getrefcount(held)\n\ + print('stable' if after == before else f'moved {before}->{after}')\n", ); + assert_eq!(out, "stable"); } #[test] -fn direct_method_dispatch_agrees() { +fn native_classes_agree() { if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { eprintln!("skipping: `data class` needs python 3.10"); return; } agree( - "direct", + "classes", "\ data class Point: x: int @@ -3297,53 +3957,36 @@ data class Point: def scaled(self, k: int) -> int: return self.total() * k - def shifted(self, d: int) -> int: - self.x = self.x + d - return self.total() - - def label(self) -> str: - return \"p\" - -data class Line: - a: Point - b: Point - - def span(self) -> int: - return self.b.total() - self.a.total() - - def widest(self, other: Line) -> int: - mine = self.span() - theirs = other.span() - if mine > theirs: - return mine - return theirs +data class Named: + label: str + count: int -def drive(a: int, b: int) -> int: - p = Point(a, b) - return p.scaled(2) + p.total() + p.shifted(1) + def shout(self) -> str: + return self.label + \"!\" -def countdown(p: Point, n: int) -> int: - total = 0 - for _ in range(n): - total = total + p.total() - return total +def make(a: int, b: int) -> object: + return Point(a, b) ", &[ - "m.Point(3, 4).scaled(5)", - "m.Point(3, 4).label()", - "m.drive(1, 2)", - "m.countdown(m.Point(2, 3), 4)", - "m.Line(m.Point(1, 1), m.Point(10, 10)).span()", - "m.Line(m.Point(1, 1), m.Point(10, 10)).widest(m.Line(m.Point(0, 0), m.Point(2, 2)))", - "[m.drive(a, a) for a in (0, -7, 10 ** 20)]", - // a python caller reaches the same method through the type object - "[getattr(m.Point(3, 4), 'scaled')(2)]", + "m.Point(3, 4).total()", + "m.Point(3, 4).scaled(10)", + "[m.Point(a, b).total() for a in (0, -1, 10 ** 20) for b in (0, 5)]", + "m.make(2, 3).total()", + "(m.Point(1, 2).x, m.Point(1, 2).y)", + "m.Named('hi', 1).shout()", + "m.Named('hi', 1).label", + // the argument *count* is checked in both builds + "[(type(e).__name__) for e in [_capture(m.Point, 1)]]", ], ); } #[test] -fn a_direct_method_call_does_not_leak() { +fn a_native_constructor_checks_its_argument_types() { + // a documented delta, not an equality: a compiled field is an unboxed + // `ByTagged`, so a `str` cannot be stored there and the check is mandatory. + // `@dataclass` does not enforce annotations at runtime, so the interpreted + // twin accepts it — the same difference `--soundness all` would close let Some((python, toolchain)) = environment() else { return; }; @@ -3351,21 +3994,12 @@ fn a_direct_method_call_does_not_leak() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_methleak"); + let dir = std::env::temp_dir().join("by_diff_ctorcheck"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -data class Holder: - label: str - - def get(self) -> str: - return self.label - -def twice(h: Holder) -> str: - return h.get() + h.get() -"; + let source = "data class Point:\n x: int\n y: int\n"; if build_source( source, - "by_diff_methleak", + "by_diff_ctorcheck", &toolchain, &dir, &Options::default(), @@ -3378,25 +4012,16 @@ def twice(h: Holder) -> str: let out = run( &python, &dir, - "import sys, by_diff_methleak as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.twice(h)\n\ - \x20 h.get()\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n", + "import by_diff_ctorcheck as m\n\ + try:\n m.Point('a', 1)\n\ + except TypeError as e:\n print('TypeError:', e)\n\ + else:\n print('accepted a str')\n", ); - assert_eq!(out, "stable"); + assert_eq!(out, "TypeError: expected int, got str"); } #[test] -fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { - // the borrow pass drops the retain/release pair around `n.inner`. getting it - // wrong is either a leak or a use-after-free, so both directions are checked: - // the label's refcount must be *stable*, and the inner holder must survive - // being dropped from python while a compiled read is in flight +fn a_native_class_has_a_fixed_layout() { let Some((python, toolchain)) = environment() else { return; }; @@ -3404,21 +4029,16 @@ fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_borrow"); + let dir = std::env::temp_dir().join("by_diff_layout"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str - -data class Nest: - inner: Holder - -def inner_label(n: Nest) -> str: - return n.inner.label +data class Point: + x: int + y: int "; if build_source( source, - "by_diff_borrow", + "by_diff_layout", &toolchain, &dir, &Options::default(), @@ -3428,65 +4048,22 @@ def inner_label(n: Nest) -> str: eprintln!("skipping: no working C toolchain"); return; } + // no `__dict__`, and no attribute outside the declared set let out = run( &python, &dir, - "import sys, by_diff_borrow as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - n = m.Nest(h)\n\ - holder_refs = sys.getrefcount(h)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.inner_label(n)\n\ - print('label', 'stable' if sys.getrefcount(label) == before else 'moved')\n\ - print('holder', 'stable' if sys.getrefcount(h) == holder_refs else 'moved')\n\ - del h\n\ - print(m.inner_label(n)[:4])\n", - ); - assert_eq!(out, "label stable\nholder stable\nxxxx"); -} - -#[test] -fn calling_a_callable_value_agrees() { - agree( - "callvalue", - "\ -def apply(f: object, a: int) -> object: - return f(a) - -def apply2(f: object, a: int, b: int) -> object: - return f(a, b) - -def indirect(a: int) -> object: - fn = abs - return fn(a) - -def shadowed(len: object, s: str) -> object: - return len(s) - -def nothing(f: object) -> object: - return f() -", - &[ - "m.apply(abs, -5)", - "m.apply(str, 5)", - "m.apply2(max, 3, 9)", - "m.indirect(-7)", - // a parameter shadowing a builtin has to win - "m.shadowed(lambda s: 'shadowed', 'abc')", - "m.nothing(dict)", - // and a value that is not callable raises the same way - "[type(e).__name__ for e in [_capture(m.apply, 3, 4)]]", - "[type(e).__name__ for e in [_capture(m.nothing, None)]]", - ], + "import by_diff_layout as m\n\ + p = m.Point(1, 2)\n\ + print(hasattr(p, '__dict__'))\n\ + try:\n p.extra = 1\n\ + except AttributeError:\n print('AttributeError')\n\ + else:\n print('accepted an undeclared attribute')\n", ); + assert_eq!(out, "False\nAttributeError"); } #[test] -fn a_borrow_survives_a_finalizer_that_runs_a_collection() { - // the borrow pass may only skip the retain where nothing can run in between. - // a `__del__` firing during the window would be the way to catch it wrong +fn a_native_class_instance_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; @@ -3494,23 +4071,16 @@ fn a_borrow_survives_a_finalizer_that_runs_a_collection() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_finalizer"); + let dir = std::env::temp_dir().join("by_diff_classleak"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Inner: +data class Holder: label: str - -data class Outer: - inner: Inner - -def read_then_call(o: Outer, f: object) -> str: - held = o.inner.label - f() - return held + o.inner.label + n: int "; if build_source( source, - "by_diff_finalizer", + "by_diff_classleak", &toolchain, &dir, &Options::default(), @@ -3520,171 +4090,65 @@ def read_then_call(o: Outer, f: object) -> str: eprintln!("skipping: no working C toolchain"); return; } + // `tp_dealloc` has to release each refcounted field let out = run( &python, &dir, - "import gc, by_diff_finalizer as m\n\ - class Noisy:\n\ - \x20 def __del__(self):\n gc.collect()\n\ - o = m.Outer(m.Inner('alpha'))\n\ - def churn():\n\ - \x20 Noisy()\n\ - \x20 o.inner = m.Inner('bravo')\n\ - for _ in range(5000):\n\ - \x20 m.read_then_call(o, churn)\n\ - print(m.read_then_call(o, lambda: None))\n", - ); - assert_eq!(out, "bravobravo"); -} - -#[test] -fn reading_a_global_as_a_value_agrees() { - agree( - "globalread", - "\ -LIMIT = 10 - -def limit() -> object: - return LIMIT - -def builtin_alias(a: int) -> object: - fn = abs - return fn(a) - -def missing() -> object: - return not_defined_anywhere -", - &[ - "m.limit()", - "m.builtin_alias(-9)", - "[type(e).__name__ for e in [_capture(m.missing)]]", - // a rebound global is observed, because the read is not cached - "[(setattr(m, 'LIMIT', 99), m.limit())[-1]]", - ], + "import sys, by_diff_classleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n m.Holder(label, 1)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", ); + assert_eq!(out, "stable"); } #[test] -fn closures_agree() { +fn a_declined_function_still_exists_and_behaves_the_same() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 11))) { + return; + } + // the interpreted fallback is what makes coverage total: a construct with no + // native lowering costs speed in that one place and nothing anywhere else agree_with_declines( - "closures", + "fallback", "\ -def make_adder(n: int) -> object: - def add(a: int) -> int: - return a + n - return add - -def make_pair(a: int, b: str) -> object: - def describe(times: int) -> str: - return b * times + str(a) - return describe - -def helper(a: int) -> int: - def double(x: int) -> int: - return x * 2 - return double(a) + double(a) - -def compose(f: object, g: object) -> object: - def both(n: int) -> object: - return f(g(n)) - return both - -def counted(n: int) -> int: - def step(a: int) -> int: - return a + n - total = 0 - for i in range(4): - total = step(total) - return total +def fast(a: int) -> int: + return a * 2 -def used_early(a: int) -> int: - if a > 0: - return later(a) - def later(x: int) -> int: - return x - return later(a) +def slow(n: int) -> int: + out = n + try: + pass + except* ValueError: + out = 0 + return out ", &[ - "m.make_adder(5)(3)", - // two closures from one function must have independent environments - "[(m.make_adder(1), m.make_adder(100))[0](0), m.make_adder(100)(0)]", - "m.make_pair(7, 'ab')(2)", - "m.helper(4)", - "m.compose(abs, lambda n: n - 10)(3)", - "m.counted(2)", - "[m.make_adder(a)(a) for a in (0, -3, 10 ** 20)]", - // the closure is a real callable python can inspect and pass around - "callable(m.make_adder(1))", - "sorted([3, 1, 2], key=m.make_adder(0))", - // and a wrong argument count still raises - "[type(e).__name__ for e in [_capture(m.make_adder(1))]]", - // python raises here, and the interpreted fallback is what reports it - "[type(e).__name__ for e in [_capture(m.used_early, 5)]]", + "m.fast(21)", + "m.slow(21)", + // the declined one is a plain python function in both builds + "type(m.slow).__name__", ], ); } #[test] -fn a_closure_does_not_leak_its_environment() { +fn a_float_module_imports_without_any_extra_runtime_module() { + // `float` transpiles through `JustFloat`, which the lazy-import pass binds + // locally. transpiling with `lazy_imports` off instead emits + // `from ty_extensions import JustFloat`, and the extension then fails to + // import at all — the embedded fallback runs at module init let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_closureleak"); + let dir = std::env::temp_dir().join("by_diff_floatimport"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def make(label: str) -> object: - def get(times: int) -> str: - return label * times - return get -"; + let source = "def area(r: float) -> float:\n return 3.0 * r * r\n"; if build_source( source, - "by_diff_closureleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the environment holds the captured `str`, and releasing the closure has to - // release the environment, which releases the field - let out = run( - &python, - &dir, - "import sys, by_diff_closureleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.make(label)(1)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n\ - held = m.make(label)\n\ - print(sys.getrefcount(label) > before)\n\ - del held\n\ - print(sys.getrefcount(label) == before)\n", - ); - assert_eq!(out, "stable\nTrue\nTrue"); -} - -#[test] -fn a_closure_environment_is_not_visible_in_the_module() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_envhidden"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def make(n: int) -> object: - def get() -> int: - return n - return get -"; - if build_source( - source, - "by_diff_envhidden", + "by_diff_floatimport", &toolchain, &dir, &Options::default(), @@ -3694,36 +4158,29 @@ def make(n: int) -> object: eprintln!("skipping: no working C toolchain"); return; } + // nothing but the extension itself is on the path let out = run( &python, &dir, - "import by_diff_envhidden as m\n\ - print([n for n in dir(m) if 'env' in n])\n\ - print(m.make(3)())\n", + "import by_diff_floatimport as m\nprint(m.area(2.0))\n", ); - assert_eq!(out, "[]\n3"); + assert_eq!(out, "12.0"); } #[test] -fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { - // the exception edge is a CFG edge, and the refcount pass used not to follow it - // — so everything the `try` body had written leaked on the exceptional path +fn a_compiled_function_is_a_c_function_object() { + // one of the few places the two builds are *meant* to differ: a natively + // compiled function is a builtin, not a python function. recorded in + // plan.md#semantic-deltas rather than asserted as equal let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_handlerleak"); + let dir = std::env::temp_dir().join("by_diff_cfunc"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def guarded(words: list[str], index: int) -> str: - held = \"held\" + words[0] - try: - return held + words[index] - except IndexError: - return held -"; + let source = "def f(a: int) -> int:\n return a\n"; if build_source( source, - "by_diff_handlerleak", + "by_diff_cfunc", &toolchain, &dir, &Options::default(), @@ -3736,259 +4193,298 @@ def guarded(words: list[str], index: int) -> str: let out = run( &python, &dir, - "import sys, by_diff_handlerleak as m\n\ - words = ['x' * 40]\n\ - label = words[0]\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.guarded(words, 9)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n\ - print(m.guarded(words, 0)[:8])\n", + "import by_diff_cfunc as m\nprint(type(m.f).__name__)\n", ); - assert_eq!(out, "stable\nheldxxxx"); + assert_eq!(out, "builtin_function_or_method"); } #[test] -fn a_mutable_capture_agrees() { - // python closes over the *variable*, so all of these depend on both frames seeing - // one cell — a copy at `def` time would give different answers for every one +fn module_level_code_runs_at_import() { agree( - "cells", + "modulelevel", "\ -def counter() -> (() -> int): - n = 0 - def get() -> int: - return n - n = 1 - return get - -def bumper() -> (() -> int): - n = 0 - def bump() -> int: - nonlocal n - n = n + 1 - return n - return bump - -def loop_closures() -> list[object]: - out = [] - i = 0 - while i < 3: - def show() -> int: - return i - out.append(show) - i = i + 1 - return out +LIMIT = 7 -def shared_pair(start: int) -> list[object]: - def read() -> int: - return start - def write(v: int) -> int: - nonlocal start - start = v - return start - return [read, write] +def under(n: int) -> int: + return n -def accumulate(values: list[int]) -> int: - total = 0 - def add(v: int) -> int: - nonlocal total - total = total + v - return total - for v in values: - add(v) - return total +def limit() -> int: + return 7 ", - &[ - "m.counter()()", - "[(b := m.bumper(), b(), b(), b())[1:]]", - "[f() for f in m.loop_closures()]", - // one cell: the write through one closure is visible through the other - "[(p := m.shared_pair(5), p[0](), p[1](9), p[0]())[1:]]", - "m.accumulate([1, 2, 3])", - "[m.accumulate([a, a]) for a in (0, -4, 10 ** 20)]", - ], + &["m.LIMIT", "m.limit()", "m.under(3)"], ); } #[test] -fn reading_a_cell_before_it_is_written_raises_the_way_python_does() { - // a cell starts unset, and NULL has to read back as an error rather than a zero - agree_with_declines( - "cellunset", - "\ -def early() -> object: - def get() -> int: - return n - out = get - n = 1 - return out +fn no_any_turns_a_gradual_decline_into_an_error() { + let Some((_, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_noany"); + let _ = std::fs::remove_dir_all(&dir); -def early_call() -> object: - def get() -> int: - return n - result = _capture_local(get) - n = 1 - return result + let source = "\ +def precise(a: int) -> int: + return a + 1 -def _capture_local(f: object) -> object: - try: - return f() - except NameError as e: - return type(e).__name__ -", - &[ - // reading it after the write is fine - "m.early()()", - // reading it before is `UnboundLocalError`, which is a `NameError` - "m.early_call()", - ], +def loose(a) -> None: + pass +"; + // by default a gradual parameter compiles — it lands on `object` + let built = build_source( + source, + "by_diff_noany", + &toolchain, + &dir, + &Options::default(), ); + match built { + Ok(built) => assert!(built.declined.is_empty(), "{:?}", built.declined), + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + } + + // `--no-any` refuses instead, and names the function + let error = build_source( + source, + "by_diff_noany", + &toolchain, + &dir, + &Options { + no_any: true, + ..Options::default() + }, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("no-any"), "{error}"); + assert!(error.contains("loose"), "{error}"); + assert!(error.contains("`a` is gradual"), "{error}"); + // a fully typed function is not blamed + assert!(!error.contains("precise"), "{error}"); } #[test] -fn a_shared_cell_does_not_leak() { - let Some((python, toolchain)) = environment() else { +fn require_native_rejects_any_decline_at_all() { + // a different question from `--no-any`: `list[int]` is not gradual, it is a + // type the compiler does not represent *yet*, so only this flag catches it + let Some((_, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_cellleak"); + let dir = std::env::temp_dir().join("by_diff_reqnative"); let _ = std::fs::remove_dir_all(&dir); + // precisely typed and still declines: `except*` has no lowering let source = "\ -def holder(label: str) -> ((str) -> str): - current = label - def swap(next: str) -> str: - nonlocal current - previous = current - current = next - return previous - return swap +def precise(a: int) -> int: + return a + 1 + +def grouped(a: int) -> int: + out = a + try: + pass + except* ValueError: + out = 0 + return out "; - if build_source( + // `--no-any` is satisfied: nothing here is gradual + match build_source( source, - "by_diff_cellleak", + "by_diff_reqnative", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); + &Options { + no_any: true, + ..Options::default() + }, + ) { + Ok(built) => assert_eq!(built.declined.len(), 1, "{:?}", built.declined), + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + } + + // `--require-native` is not + let error = build_source( + source, + "by_diff_reqnative", + &toolchain, + &dir, + &Options { + require_native: true, + ..Options::default() + }, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("require-native"), "{error}"); + assert!(error.contains("`except*`"), "{error}"); +} + +#[test] +fn no_any_accepts_a_fully_typed_module() { + let Some((_, toolchain)) = environment() else { return; + }; + let dir = std::env::temp_dir().join("by_diff_noany_ok"); + let _ = std::fs::remove_dir_all(&dir); + let source = "def f(a: int, b: int) -> int:\n return a * b\n"; + match build_source( + source, + "by_diff_noany_ok", + &toolchain, + &dir, + &Options { + no_any: true, + ..Options::default() + }, + ) { + Ok(built) => assert!(built.declined.is_empty()), + Err(error) => eprintln!("skipping: no working C toolchain ({error})"), } - // the cell holds a reference and a write must release the old one. the live - // closure has to be dropped before measuring, or its own hold on the cell reads - // as a leak - let out = run( - &python, +} + +#[test] +fn optimized_output_still_agrees_with_the_interpreter() { + // copy propagation and infallibility both rewrite the IR — the differential + // legs are what say the rewrites preserved the program + agree( + "optimized", + "\ +def area(r: float) -> float: + scaled = r * r + return 3.5 * scaled + +def compose(a: float, b: float) -> float: + x = area(a) + y = area(b) + return x + y +", + &[ + "m.area(2.0)", + "m.compose(1.0, 2.0)", + "[m.compose(a, a) for a in (0.0, -1.5, 3.25)]", + ], + ); +} + +#[test] +fn a_declined_function_is_reported_with_a_reason() { + let Some((_, toolchain)) = environment() else { + return; + }; + let dir: PathBuf = std::env::temp_dir().join("by_diff_declined"); + let _ = std::fs::remove_dir_all(&dir); + + let source = "\ +def fast(a: int) -> int: + return a + 1 + +def slow(a: int) -> None: + try: + pass + except* ValueError: + pass +"; + let Ok(built) = build_source( + source, + "by_diff_declined", + &toolchain, &dir, - "import gc, sys, by_diff_cellleak as m\n\ - label = 'x' * 40\n\ - other = 'y' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 swap = m.holder(label)\n\ - \x20 swap(other)\n\ - \x20 swap(label)\n\ - del swap\n\ - print('refs', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - gc.collect()\n\ - objects = len(gc.get_objects())\n\ - for _ in range(2000):\n\ - \x20 held = m.holder(label)\n\ - \x20 held(other)\n\ - del held\n\ - gc.collect()\n\ - print('envs', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", + &Options::default(), + ) else { + eprintln!("skipping: no working C toolchain"); + return; + }; + assert_eq!(built.declined.len(), 1); + assert_eq!(built.declined[0].name, "slow"); + assert!( + built.declined[0].reason.contains("`except*`"), + "{:?}", + built.declined[0] ); - assert_eq!(out, "refs stable\nenvs stable"); } #[test] -fn generators_agree() { - agree_with_declines( - "generators", +fn field_access_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree( + "fields", "\ -def counted(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 +data class Point: + x: int + y: int -def three() -> object: - yield 1 - yield 2 - yield 3 + def total(self) -> int: + return self.x + self.y -def accumulating(words: list[str]) -> object: - seen = \"\" - for w in words: - seen = seen + w - yield seen + def shift(self, d: int) -> int: + self.x = self.x + d + return self.x -def pairs(xs: list[int], ys: list[int]) -> object: - for a in xs: - for b in ys: - yield a * b +data class Line: + a: Point + b: Point -def nothing() -> object: - if False: - yield 1 + def span(self) -> int: + return self.b.x - self.a.x -def early(n: int) -> object: - yield n - return + def retarget(self, p: Point) -> int: + self.b = p + return self.span() -def echoing() -> object: - total = 0 - while True: - got = yield total - total = total + 1 +def sum_of(p: Point) -> int: + return p.x + p.y + +def bump_twice(p: Point) -> int: + p.shift(1) + p.shift(2) + return p.x ", &[ - "list(m.counted(4))", - "list(m.three())", - "list(m.accumulating(['a', 'bb', 'ccc']))", - "list(m.pairs([1, 2, 3], [10, 20]))", - "list(m.nothing())", - "list(m.early(7))", - // arbitrary precision survives the suspension - "list(m.counted(3))[-1] + 10 ** 20", - // partial consumption, then more - "[(g := m.counted(5), next(g), next(g), list(g))[1:]]", - // it is a real iterator, so everything that takes one works - "sum(m.counted(5))", - "sorted(m.three(), reverse=True)", - "[x for x in m.counted(3) if x]", - "list(zip(m.counted(3), m.three()))", - // exhaustion keeps raising - "[type(e).__name__ for e in [_capture(next, m.nothing())]]", - "[(g := m.early(1), next(g), type(_capture(next, g)).__name__, type(_capture(next, g)).__name__)[2:]]", - // `send` is what the `yield` expression evaluates to - "[(e := m.echoing(), next(e), e.send(9), e.send(9))[1:]]", - // `close` exhausts it - "[(g := m.counted(9), next(g), g.close(), type(_capture(next, g)).__name__)[3:]]", + "m.Point(3, 4).total()", + "m.sum_of(m.Point(3, 4))", + "m.Point(3, 4).shift(10)", + "m.bump_twice(m.Point(0, 0))", + "m.Line(m.Point(1, 2), m.Point(10, 20)).span()", + "m.Line(m.Point(1, 2), m.Point(10, 20)).a.y", + "m.Line(m.Point(1, 2), m.Point(10, 20)).retarget(m.Point(100, 0))", + // a write through the python-visible setter, then a native read + "[(p := m.Point(1, 2), setattr(p, 'x', 9), p.total())[-1]]", + "[m.sum_of(m.Point(a, b)) for a in (0, -1, 10 ** 20) for b in (0, 5)]", ], ); } #[test] -fn a_generator_is_a_real_iterator_to_python() { +fn a_field_setter_checks_its_value() { + // the same documented delta as the constructor: an unboxed field cannot hold + // the wrong representation, and `@dataclass` does not check assignments let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_geniter"); + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_setcheck"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -def counted(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 +data class Point: + x: int + y: int + +frozen data class Fixed: + n: int "; if build_source( source, - "by_diff_geniter", + "by_diff_setcheck", &toolchain, &dir, &Options::default(), @@ -4001,679 +4497,2316 @@ def counted(n: int) -> object: let out = run( &python, &dir, - "import by_diff_geniter as m\n\ - g = m.counted(3)\n\ - print(iter(g) is g)\n\ - print(hasattr(g, '__next__'), hasattr(g, 'send'), hasattr(g, 'close'))\n\ - print(list(g))\n", - ); - assert_eq!(out, "True\nTrue True True\n[0, 1, 2]"); + "import by_diff_setcheck as m\n\ + p = m.Point(1, 2)\n\ + p.x = 5\n\ + print(p.x)\n\ + for attempt in (lambda: setattr(p, 'x', 'a'), lambda: delattr(p, 'x'), lambda: setattr(m.Fixed(1), 'n', 2)):\n\ + \x20 try:\n attempt()\n\ + \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ + \x20 else:\n print('accepted')\n", + ); + assert_eq!(out, "5\nTypeError\nAttributeError\nAttributeError"); } -/// the generator shapes every leak test below drives, built once -/// -/// `require_native` is what makes the answers the *compiled* generator's: a -/// declined function would run from its interpreted definition and leak nothing, -/// so the test would pass without exercising anything -fn leak_module(tag: &'static str) -> Option<(String, std::path::PathBuf)> { - let (python, toolchain) = environment()?; - let dir = std::env::temp_dir().join(tag); +#[test] +fn a_class_typed_argument_is_checked_at_the_boundary() { + // a compiled parameter typed as a native class is a pointer to its struct, + // so python handing over anything else has to be caught here — the + // alternative is a wild pointer dereference + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_argcheck"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Boom(Exception): - pass - -def repeat(label: str, times: int) -> object: - i = 0 - while i < times: - yield label - i = i + 1 - -def guarded(times: int) -> object: - i = 0 - while i < times: - try: - yield i - except Boom: - yield -1 - i = i + 1 +data class Point: + x: int -def in_handler(times: int) -> object: - i = 0 - while i < times: - try: - raise Boom() - except Boom: - yield i - i = i + 1 +def read(p: Point) -> int: + return p.x "; - let options = Options { - require_native: true, - ..Options::default() - }; - if build_source(source, tag, &toolchain, &dir, &options).is_err() { + if build_source( + source, + "by_diff_argcheck", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { eprintln!("skipping: no working C toolchain"); - return None; + return; } - Some((python, dir)) + let out = run( + &python, + &dir, + "import by_diff_argcheck as m\n\ + print(m.read(m.Point(7)))\n\ + for bad in ('a', 1, None):\n\ + \x20 try:\n m.read(bad)\n\ + \x20 except TypeError as e:\n print(e)\n\ + \x20 else:\n print('accepted')\n", + ); + assert_eq!( + out, + "7\nexpected by_diff_argcheck.Point, got str\n\ + expected by_diff_argcheck.Point, got int\n\ + expected by_diff_argcheck.Point, got NoneType" + ); } -/// the two counters every leak test here uses -/// -/// a *total* object count is too noisy to see one leaked object per iteration -/// until the iteration count is large, and a count of some unrelated object is -/// blind to it entirely — so each of these watches the object that would actually -/// be leaked: instances of one class, and the reference count of one instance -const LEAK_INSTRUMENTS: &str = "\ -import gc, sys -def live(kind): - gc.collect() - return sum(1 for o in gc.get_objects() if type(o) is kind) -def leaked(kind, once): - base = live(kind) - for _ in range(50): - once() - return live(kind) - base -"; - #[test] -fn a_generator_does_not_leak_its_state() { - let Some((python, dir)) = leak_module("by_diff_genleak") else { +fn a_field_read_does_not_leak() { + let Some((python, toolchain)) = environment() else { return; }; - // the state object holds every parameter, and dropping it must release them — - // including when the generator is abandoned part-way through. - // - // watching the *label* alone is what let a leaked `GeneratorExit` through for - // as long as it did: the parameter is released either way, so the count below - // it is the one that moves + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_fieldleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + +def label_of(h: Holder) -> str: + return h.label + +data class Nest: + inner: Holder + +def inner_label(n: Nest) -> str: + return n.inner.label +"; + if build_source( + source, + "by_diff_fieldleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // a field read hands back a *borrowed* reference, so the register that takes + // it must retain — and release. `str` is not gc-tracked, so this measures the + // refcount rather than the object count let out = run( &python, &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 list(m.repeat(label, 3))\n\ - print('drained', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.repeat(label, 9)\n\ - \x20 next(g)\n\ - del g\n\ - gc.collect()\n\ - print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - def abandon():\n\ - \x20 g = m.repeat(label, 9)\n\ - \x20 next(g)\n\ - print('exit objects', leaked(GeneratorExit, abandon))\n" - ), + "import sys, by_diff_fieldleak as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + n = m.Nest(h)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.label_of(h)\n\ + \x20 m.inner_label(n)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n", ); - assert_eq!(out, "drained stable\nabandoned stable\nexit objects 0"); + assert_eq!(out, "stable"); } -/// finalising a suspended generator throws `GeneratorExit` in, and the unwind has -/// to release it — the frame does not own what it hands to the error state #[test] -fn ending_a_generator_releases_the_generator_exit() { - let Some((python, dir)) = leak_module("by_diff_genexit") else { +fn a_constructor_result_is_used_natively() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); return; - }; - let out = run( - &python, - &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genexit as m\n\ - def abandoned():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - def exhausted():\n\ - \x20 list(m.repeat('x', 3))\n\ - def closed():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - \x20 g.close()\n\ - def in_handler():\n\ - \x20 g = m.in_handler(9)\n\ - \x20 next(g)\n\ - print('abandoned', leaked(GeneratorExit, abandoned))\n\ - print('exhausted', leaked(GeneratorExit, exhausted))\n\ - print('closed', leaked(GeneratorExit, closed))\n\ - print('in handler', leaked(GeneratorExit, in_handler))\n" - ), + } + agree( + "ctornative", + "\ +data class Point: + x: int + y: int + + def total(self) -> int: + return self.x + self.y + +data class Line: + a: Point + b: Point + +def diag(a: int) -> int: + return Point(a, a).x + +def build(a: int, b: int) -> int: + return Line(Point(a, b), Point(b, a)).b.x + +def widened(a: int) -> object: + return Point(a, a) + +def chained(a: int) -> int: + p = Point(a, a + 1) + q = Point(p.y, p.x) + return q.total() + p.total() +", + &[ + "m.diag(7)", + "m.build(3, 9)", + "m.widened(4).total()", + "m.chained(5)", + "[m.chained(a) for a in (0, -3, 10 ** 20)]", + ], + ); +} + +#[test] +fn a_declined_callee_does_not_break_the_build() { + // the whole module used to fail to compile: the emitted call named a symbol + // that was never defined + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree_with_declines( + "declinechain", + "\ +def helper[T](a: T) -> T: + return a + +def caller(a: int) -> int: + return helper(a) + a + +data class Point: + x: int + + def bad[T](self, a: T) -> int: + return self.x + +def read(p: Point) -> int: + return p.x + +def alone(a: int) -> int: + return a + 1 +", + &[ + "m.helper(1)", + "m.caller(2)", + "m.read(m.Point(5))", + "m.Point(5).bad(1)", + "m.alone(3)", + ], + ); +} + +#[test] +fn direct_method_dispatch_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree( + "direct", + "\ +data class Point: + x: int + y: int + + def total(self) -> int: + return self.x + self.y + + def scaled(self, k: int) -> int: + return self.total() * k + + def shifted(self, d: int) -> int: + self.x = self.x + d + return self.total() + + def label(self) -> str: + return \"p\" + +data class Line: + a: Point + b: Point + + def span(self) -> int: + return self.b.total() - self.a.total() + + def widest(self, other: Line) -> int: + mine = self.span() + theirs = other.span() + if mine > theirs: + return mine + return theirs + +def drive(a: int, b: int) -> int: + p = Point(a, b) + return p.scaled(2) + p.total() + p.shifted(1) + +def countdown(p: Point, n: int) -> int: + total = 0 + for _ in range(n): + total = total + p.total() + return total +", + &[ + "m.Point(3, 4).scaled(5)", + "m.Point(3, 4).label()", + "m.drive(1, 2)", + "m.countdown(m.Point(2, 3), 4)", + "m.Line(m.Point(1, 1), m.Point(10, 10)).span()", + "m.Line(m.Point(1, 1), m.Point(10, 10)).widest(m.Line(m.Point(0, 0), m.Point(2, 2)))", + "[m.drive(a, a) for a in (0, -7, 10 ** 20)]", + // a python caller reaches the same method through the type object + "[getattr(m.Point(3, 4), 'scaled')(2)]", + ], + ); +} + +#[test] +fn a_direct_method_call_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_methleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + + def get(self) -> str: + return self.label + +def twice(h: Holder) -> str: + return h.get() + h.get() +"; + if build_source( + source, + "by_diff_methleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_methleak as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.twice(h)\n\ + \x20 h.get()\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n", + ); + assert_eq!(out, "stable"); +} + +#[test] +fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { + // the borrow pass drops the retain/release pair around `n.inner`. getting it + // wrong is either a leak or a use-after-free, so both directions are checked: + // the label's refcount must be *stable*, and the inner holder must survive + // being dropped from python while a compiled read is in flight + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_borrow"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + +data class Nest: + inner: Holder + +def inner_label(n: Nest) -> str: + return n.inner.label +"; + if build_source( + source, + "by_diff_borrow", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_borrow as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + n = m.Nest(h)\n\ + holder_refs = sys.getrefcount(h)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.inner_label(n)\n\ + print('label', 'stable' if sys.getrefcount(label) == before else 'moved')\n\ + print('holder', 'stable' if sys.getrefcount(h) == holder_refs else 'moved')\n\ + del h\n\ + print(m.inner_label(n)[:4])\n", + ); + assert_eq!(out, "label stable\nholder stable\nxxxx"); +} + +#[test] +fn calling_a_callable_value_agrees() { + agree( + "callvalue", + "\ +def apply(f: object, a: int) -> object: + return f(a) + +def apply2(f: object, a: int, b: int) -> object: + return f(a, b) + +def indirect(a: int) -> object: + fn = abs + return fn(a) + +def shadowed(len: object, s: str) -> object: + return len(s) + +def nothing(f: object) -> object: + return f() +", + &[ + "m.apply(abs, -5)", + "m.apply(str, 5)", + "m.apply2(max, 3, 9)", + "m.indirect(-7)", + // a parameter shadowing a builtin has to win + "m.shadowed(lambda s: 'shadowed', 'abc')", + "m.nothing(dict)", + // and a value that is not callable raises the same way + "[type(e).__name__ for e in [_capture(m.apply, 3, 4)]]", + "[type(e).__name__ for e in [_capture(m.nothing, None)]]", + ], + ); +} + +#[test] +fn a_borrow_survives_a_finalizer_that_runs_a_collection() { + // the borrow pass may only skip the retain where nothing can run in between. + // a `__del__` firing during the window would be the way to catch it wrong + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_finalizer"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Inner: + label: str + +data class Outer: + inner: Inner + +def read_then_call(o: Outer, f: object) -> str: + held = o.inner.label + f() + return held + o.inner.label +"; + if build_source( + source, + "by_diff_finalizer", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import gc, by_diff_finalizer as m\n\ + class Noisy:\n\ + \x20 def __del__(self):\n gc.collect()\n\ + o = m.Outer(m.Inner('alpha'))\n\ + def churn():\n\ + \x20 Noisy()\n\ + \x20 o.inner = m.Inner('bravo')\n\ + for _ in range(5000):\n\ + \x20 m.read_then_call(o, churn)\n\ + print(m.read_then_call(o, lambda: None))\n", + ); + assert_eq!(out, "bravobravo"); +} + +#[test] +fn reading_a_global_as_a_value_agrees() { + agree( + "globalread", + "\ +LIMIT = 10 + +def limit() -> object: + return LIMIT + +def builtin_alias(a: int) -> object: + fn = abs + return fn(a) + +def missing() -> object: + return not_defined_anywhere +", + &[ + "m.limit()", + "m.builtin_alias(-9)", + "[type(e).__name__ for e in [_capture(m.missing)]]", + // a rebound global is observed, because the read is not cached + "[(setattr(m, 'LIMIT', 99), m.limit())[-1]]", + ], + ); +} + +#[test] +fn closures_agree() { + agree_with_declines( + "closures", + "\ +def make_adder(n: int) -> object: + def add(a: int) -> int: + return a + n + return add + +def make_pair(a: int, b: str) -> object: + def describe(times: int) -> str: + return b * times + str(a) + return describe + +def helper(a: int) -> int: + def double(x: int) -> int: + return x * 2 + return double(a) + double(a) + +def compose(f: object, g: object) -> object: + def both(n: int) -> object: + return f(g(n)) + return both + +def counted(n: int) -> int: + def step(a: int) -> int: + return a + n + total = 0 + for i in range(4): + total = step(total) + return total + +def used_early(a: int) -> int: + if a > 0: + return later(a) + def later(x: int) -> int: + return x + return later(a) +", + &[ + "m.make_adder(5)(3)", + // two closures from one function must have independent environments + "[(m.make_adder(1), m.make_adder(100))[0](0), m.make_adder(100)(0)]", + "m.make_pair(7, 'ab')(2)", + "m.helper(4)", + "m.compose(abs, lambda n: n - 10)(3)", + "m.counted(2)", + "[m.make_adder(a)(a) for a in (0, -3, 10 ** 20)]", + // the closure is a real callable python can inspect and pass around + "callable(m.make_adder(1))", + "sorted([3, 1, 2], key=m.make_adder(0))", + // and a wrong argument count still raises + "[type(e).__name__ for e in [_capture(m.make_adder(1))]]", + // python raises here, and the interpreted fallback is what reports it + "[type(e).__name__ for e in [_capture(m.used_early, 5)]]", + ], + ); +} + +#[test] +fn a_closure_does_not_leak_its_environment() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_closureleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def make(label: str) -> object: + def get(times: int) -> str: + return label * times + return get +"; + if build_source( + source, + "by_diff_closureleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the environment holds the captured `str`, and releasing the closure has to + // release the environment, which releases the field + let out = run( + &python, + &dir, + "import sys, by_diff_closureleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.make(label)(1)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n\ + held = m.make(label)\n\ + print(sys.getrefcount(label) > before)\n\ + del held\n\ + print(sys.getrefcount(label) == before)\n", + ); + assert_eq!(out, "stable\nTrue\nTrue"); +} + +#[test] +fn a_closure_environment_is_not_visible_in_the_module() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_envhidden"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def make(n: int) -> object: + def get() -> int: + return n + return get +"; + if build_source( + source, + "by_diff_envhidden", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_envhidden as m\n\ + print([n for n in dir(m) if 'env' in n])\n\ + print(m.make(3)())\n", + ); + assert_eq!(out, "[]\n3"); +} + +#[test] +fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { + // the exception edge is a CFG edge, and the refcount pass used not to follow it + // — so everything the `try` body had written leaked on the exceptional path + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_handlerleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def guarded(words: list[str], index: int) -> str: + held = \"held\" + words[0] + try: + return held + words[index] + except IndexError: + return held +"; + if build_source( + source, + "by_diff_handlerleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_handlerleak as m\n\ + words = ['x' * 40]\n\ + label = words[0]\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.guarded(words, 9)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n\ + print(m.guarded(words, 0)[:8])\n", + ); + assert_eq!(out, "stable\nheldxxxx"); +} + +#[test] +fn a_mutable_capture_agrees() { + // python closes over the *variable*, so all of these depend on both frames seeing + // one cell — a copy at `def` time would give different answers for every one + agree( + "cells", + "\ +def counter() -> (() -> int): + n = 0 + def get() -> int: + return n + n = 1 + return get + +def bumper() -> (() -> int): + n = 0 + def bump() -> int: + nonlocal n + n = n + 1 + return n + return bump + +def loop_closures() -> list[object]: + out = [] + i = 0 + while i < 3: + def show() -> int: + return i + out.append(show) + i = i + 1 + return out + +def shared_pair(start: int) -> list[object]: + def read() -> int: + return start + def write(v: int) -> int: + nonlocal start + start = v + return start + return [read, write] + +def accumulate(values: list[int]) -> int: + total = 0 + def add(v: int) -> int: + nonlocal total + total = total + v + return total + for v in values: + add(v) + return total +", + &[ + "m.counter()()", + "[(b := m.bumper(), b(), b(), b())[1:]]", + "[f() for f in m.loop_closures()]", + // one cell: the write through one closure is visible through the other + "[(p := m.shared_pair(5), p[0](), p[1](9), p[0]())[1:]]", + "m.accumulate([1, 2, 3])", + "[m.accumulate([a, a]) for a in (0, -4, 10 ** 20)]", + ], + ); +} + +#[test] +fn reading_a_cell_before_it_is_written_raises_the_way_python_does() { + // a cell starts unset, and NULL has to read back as an error rather than a zero + agree_with_declines( + "cellunset", + "\ +def early() -> object: + def get() -> int: + return n + out = get + n = 1 + return out + +def early_call() -> object: + def get() -> int: + return n + result = _capture_local(get) + n = 1 + return result + +def _capture_local(f: object) -> object: + try: + return f() + except NameError as e: + return type(e).__name__ +", + &[ + // reading it after the write is fine + "m.early()()", + // reading it before is `UnboundLocalError`, which is a `NameError` + "m.early_call()", + ], + ); +} + +#[test] +fn a_shared_cell_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_cellleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def holder(label: str) -> ((str) -> str): + current = label + def swap(next: str) -> str: + nonlocal current + previous = current + current = next + return previous + return swap +"; + if build_source( + source, + "by_diff_cellleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the cell holds a reference and a write must release the old one. the live + // closure has to be dropped before measuring, or its own hold on the cell reads + // as a leak + let out = run( + &python, + &dir, + "import gc, sys, by_diff_cellleak as m\n\ + label = 'x' * 40\n\ + other = 'y' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 swap = m.holder(label)\n\ + \x20 swap(other)\n\ + \x20 swap(label)\n\ + del swap\n\ + print('refs', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + gc.collect()\n\ + objects = len(gc.get_objects())\n\ + for _ in range(2000):\n\ + \x20 held = m.holder(label)\n\ + \x20 held(other)\n\ + del held\n\ + gc.collect()\n\ + print('envs', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", + ); + assert_eq!(out, "refs stable\nenvs stable"); +} + +#[test] +fn generators_agree() { + agree_with_declines( + "generators", + "\ +def counted(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 + +def three() -> object: + yield 1 + yield 2 + yield 3 + +def accumulating(words: list[str]) -> object: + seen = \"\" + for w in words: + seen = seen + w + yield seen + +def pairs(xs: list[int], ys: list[int]) -> object: + for a in xs: + for b in ys: + yield a * b + +def nothing() -> object: + if False: + yield 1 + +def early(n: int) -> object: + yield n + return + +def echoing() -> object: + total = 0 + while True: + got = yield total + total = total + 1 +", + &[ + "list(m.counted(4))", + "list(m.three())", + "list(m.accumulating(['a', 'bb', 'ccc']))", + "list(m.pairs([1, 2, 3], [10, 20]))", + "list(m.nothing())", + "list(m.early(7))", + // arbitrary precision survives the suspension + "list(m.counted(3))[-1] + 10 ** 20", + // partial consumption, then more + "[(g := m.counted(5), next(g), next(g), list(g))[1:]]", + // it is a real iterator, so everything that takes one works + "sum(m.counted(5))", + "sorted(m.three(), reverse=True)", + "[x for x in m.counted(3) if x]", + "list(zip(m.counted(3), m.three()))", + // exhaustion keeps raising + "[type(e).__name__ for e in [_capture(next, m.nothing())]]", + "[(g := m.early(1), next(g), type(_capture(next, g)).__name__, type(_capture(next, g)).__name__)[2:]]", + // `send` is what the `yield` expression evaluates to + "[(e := m.echoing(), next(e), e.send(9), e.send(9))[1:]]", + // `close` exhausts it + "[(g := m.counted(9), next(g), g.close(), type(_capture(next, g)).__name__)[3:]]", + ], + ); +} + +#[test] +fn a_generator_is_a_real_iterator_to_python() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_geniter"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def counted(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 +"; + if build_source( + source, + "by_diff_geniter", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_geniter as m\n\ + g = m.counted(3)\n\ + print(iter(g) is g)\n\ + print(hasattr(g, '__next__'), hasattr(g, 'send'), hasattr(g, 'close'))\n\ + print(list(g))\n", + ); + assert_eq!(out, "True\nTrue True True\n[0, 1, 2]"); +} + +/// the generator shapes every leak test below drives, built once +/// +/// `require_native` is what makes the answers the *compiled* generator's: a +/// declined function would run from its interpreted definition and leak nothing, +/// so the test would pass without exercising anything +fn leak_module(tag: &'static str) -> Option<(String, std::path::PathBuf)> { + let (python, toolchain) = environment()?; + let dir = std::env::temp_dir().join(tag); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Boom(Exception): + pass + +def repeat(label: str, times: int) -> object: + i = 0 + while i < times: + yield label + i = i + 1 + +def guarded(times: int) -> object: + i = 0 + while i < times: + try: + yield i + except Boom: + yield -1 + i = i + 1 + +def in_handler(times: int) -> object: + i = 0 + while i < times: + try: + raise Boom() + except Boom: + yield i + i = i + 1 +"; + let options = Options { + require_native: true, + ..Options::default() + }; + if build_source(source, tag, &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return None; + } + Some((python, dir)) +} + +/// the two counters every leak test here uses +/// +/// a *total* object count is too noisy to see one leaked object per iteration +/// until the iteration count is large, and a count of some unrelated object is +/// blind to it entirely — so each of these watches the object that would actually +/// be leaked: instances of one class, and the reference count of one instance +const LEAK_INSTRUMENTS: &str = "\ +import gc, sys +def live(kind): + gc.collect() + return sum(1 for o in gc.get_objects() if type(o) is kind) +def leaked(kind, once): + base = live(kind) + for _ in range(50): + once() + return live(kind) - base +"; + +#[test] +fn a_generator_does_not_leak_its_state() { + let Some((python, dir)) = leak_module("by_diff_genleak") else { + return; + }; + // the state object holds every parameter, and dropping it must release them — + // including when the generator is abandoned part-way through. + // + // watching the *label* alone is what let a leaked `GeneratorExit` through for + // as long as it did: the parameter is released either way, so the count below + // it is the one that moves + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 list(m.repeat(label, 3))\n\ + print('drained', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.repeat(label, 9)\n\ + \x20 next(g)\n\ + del g\n\ + gc.collect()\n\ + print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + def abandon():\n\ + \x20 g = m.repeat(label, 9)\n\ + \x20 next(g)\n\ + print('exit objects', leaked(GeneratorExit, abandon))\n" + ), + ); + assert_eq!(out, "drained stable\nabandoned stable\nexit objects 0"); +} + +/// finalising a suspended generator throws `GeneratorExit` in, and the unwind has +/// to release it — the frame does not own what it hands to the error state +#[test] +fn ending_a_generator_releases_the_generator_exit() { + let Some((python, dir)) = leak_module("by_diff_genexit") else { + return; + }; + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genexit as m\n\ + def abandoned():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + def exhausted():\n\ + \x20 list(m.repeat('x', 3))\n\ + def closed():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + \x20 g.close()\n\ + def in_handler():\n\ + \x20 g = m.in_handler(9)\n\ + \x20 next(g)\n\ + print('abandoned', leaked(GeneratorExit, abandoned))\n\ + print('exhausted', leaked(GeneratorExit, exhausted))\n\ + print('closed', leaked(GeneratorExit, closed))\n\ + print('in handler', leaked(GeneratorExit, in_handler))\n" + ), + ); + assert_eq!(out, "abandoned 0\nexhausted 0\nclosed 0\nin handler 0"); +} + +/// a `throw` raises at the suspension point, and whether a handler catches it or +/// it comes back out, nothing may hold a reference afterwards +#[test] +fn throwing_into_a_generator_releases_the_exception() { + let Some((python, dir)) = leak_module("by_diff_genthrow") else { + return; + }; + // two instruments over the same shapes: one instance thrown again and again, + // whose own reference count moves even though the object stays reachable, and a + // fresh instance per call, which a retained reference keeps alive as a countable + // object. + // + // the exception is a global rather than a parameter on purpose. a caught + // exception's traceback holds the frames it passed through, so a local naming it + // is a reference the *test* keeps — 50 of them, in both builds alike + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genthrow as m\n\ + def caught():\n\ + \x20 g = m.guarded(9)\n\ + \x20 next(g)\n\ + \x20 g.throw(thrown)\n\ + def uncaught():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + \x20 try:\n\ + \x20 g.throw(thrown)\n\ + \x20 except m.Boom:\n\ + \x20 pass\n\ + def in_handler():\n\ + \x20 g = m.in_handler(9)\n\ + \x20 next(g)\n\ + def fresh(once):\n\ + \x20 def run():\n\ + \x20 global thrown\n\ + \x20 thrown = m.Boom()\n\ + \x20 once()\n\ + \x20 thrown = None\n\ + \x20 return run\n\ + thrown = m.Boom()\n\ + boom = thrown\n\ + gc.collect()\n\ + held = sys.getrefcount(boom)\n\ + for _ in range(50):\n\ + \x20 caught()\n\ + \x20 uncaught()\n\ + gc.collect()\n\ + print('thrown refs', sys.getrefcount(boom) - held)\n\ + print('caught', leaked(m.Boom, fresh(caught)))\n\ + print('uncaught', leaked(m.Boom, fresh(uncaught)))\n\ + print('raised inside', leaked(m.Boom, in_handler))\n" + ), + ); + assert_eq!(out, "thrown refs 0\ncaught 0\nuncaught 0\nraised inside 0"); +} + +/// the same rule outside a generator, which is where it is stated: a re-raise +/// hands the exception to the interpreter and keeps nothing +#[test] +fn a_re_raise_does_not_retain_the_exception() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_reraiseleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Boom(Exception): + pass + +def bare(x: int) -> int: + try: + raise Boom() + except Boom: + raise + +def unmatched(x: int) -> int: + try: + raise Boom() + except ValueError: + return 1 + return 2 + +def through_finally(x: int) -> int: + try: + raise Boom() + finally: + x = x + 1 +"; + let options = Options { + require_native: true, + ..Options::default() + }; + if build_source(source, "by_diff_reraiseleak", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_reraiseleak as m\n\ + def raising(fn):\n\ + \x20 def once():\n\ + \x20 try:\n\ + \x20 fn(1)\n\ + \x20 except m.Boom:\n\ + \x20 pass\n\ + \x20 return once\n\ + for name in ('bare', 'unmatched', 'through_finally'):\n\ + \x20 print(name, leaked(m.Boom, raising(getattr(m, name))))\n" + ), + ); + assert_eq!(out, "bare 0\nunmatched 0\nthrough_finally 0"); +} + +#[test] +fn delegation_agrees() { + agree_with_declines( + "delegation", + "\ +def inner(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 + return n * 100 + +def outer(n: int) -> object: + got = yield from inner(n) + yield got + +def chained(xs: list[int]) -> object: + yield from xs + yield from xs + +def nested(n: int) -> object: + yield from outer(n) +", + &[ + "list(m.inner(3))", + // the delegated `return` value is what the expression evaluates to + "list(m.outer(3))", + "list(m.chained([1, 2]))", + "list(m.nested(2))", + "list(m.chained([]))", + "sum(m.outer(4))", + // and the inner generator's own `StopIteration` value reaches the consumer + "[(g := m.inner(1), next(g), _capture(next, g).value)[2:]]", + ], + ); +} + +#[test] +fn async_await_agrees() { + agree_with_declines( + "asyncawait", + "\ +async def plain(n: int) -> int: + return n * 2 + +async def chained(n: int) -> int: + a = await plain(n) + b = await plain(a) + return b + 1 + +async def nothing() -> None: + pass + +async def forwards(n: int) -> int: + return await chained(n) +", + &[ + "__import__('asyncio').run(m.plain(5))", + "__import__('asyncio').run(m.chained(3))", + "__import__('asyncio').run(m.nothing())", + "__import__('asyncio').run(m.forwards(1))", + "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", + ], + ); +} + +#[test] +fn the_await_protocol_agrees() { + // an `await` reaches its object through `__await__` and drives what comes back + // by sending into it. both halves have edges worth pinning: what counts as + // awaitable at all, and what a finished delegation's value *is* + agree( + "awaitproto", + "\ +async def awaited(x: object) -> object: + return await x + +async def twice(x: object, y: object) -> object: + a = await x + b = await y + return (a, b) + +async def guarded(x: object) -> object: + try: + return await x + except ValueError as e: + return 'caught ' + str(e) + +def delegated(xs: object) -> object: + got = yield from xs + yield got + +def returns(v: object) -> object: + return v + yield + +async def returns_async(v: object) -> object: + return v + +def redelegated(v: object) -> object: + got = yield from returns(v) + yield got +", + &[ + // the *other* half of the same rule: what a compiled frame's own return + // value becomes on the way out. `StopIteration` reads its argument as an + // argument *list*, so a tuple would arrive spread and an exception + // instance would be raised in place of the `StopIteration` itself + "[_value(m.returns(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2], 'ab')]", + "[_run(m.returns_async(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2])]", + "[list(m.redelegated(v)) for v in ((1, 2), (), (1,), 5, None)]", + "[_run(m.awaited(m.returns_async(v))) for v in ((1, 2), (), 5, None)]", + "_value(m.returns(ValueError('x')))", + "_value(m.returns(StopIteration(9)))", + "_run(m.returns_async(StopIteration(9)))", + // and the exception it rides on is shaped the way python shapes it + "[(lambda e: (repr(e), e.args))(_capture(next, m.returns(v))) \ + for v in ((1, 2), (), 5, None)]", + // an await that completes without ever suspending — the common case + "_run(m.awaited(_Ready(7)))", + "_run(m.twice(_Ready(1), _Ready(2)))", + // one that really suspends, driven by a loop that has to resume it + "_run(m.awaited(_sleeps(9)))", + "_run(m.awaited(_Suspends(11)))", + "_run(m.twice(_Ready(1), _sleeps(2)))", + "_run(m.twice(_Suspends(1), _Suspends(2)))", + // a bare `StopIteration` carries `None`, and a subclass carries its own + "_run(m.awaited(_Raises(StopIteration())))", + "_run(m.awaited(_Raises(StopIteration(3))))", + "_run(m.awaited(_Raises(_Sub(5))))", + // python reads the exception's field, not its `value` attribute + "_run(m.awaited(_Raises(_Shadowed(7))))", + "_Shadowed(7).value", + // a raised *type* rather than an instance still has to become one + "_run(m.awaited(_Raises(StopIteration)))", + // normalising a tuple argument would take only its first element + "_run(m.awaited(_Raises(StopIteration((1, 2)))))", + // anything else escaping the awaited object is not a return value + "_capture_async(m.awaited, _throws(ValueError))", + "_capture_async(m.awaited, _throws(KeyError))", + "_run(m.guarded(_throws(ValueError)))", + "_capture_async(m.guarded, _throws(KeyError))", + // and what is not awaitable at all says so, about the right object + "_capture_async(m.awaited, 5)", + "_capture_async(m.awaited, _counting(1))", + "_capture_async(m.awaited, _NotIter())", + "_capture_async(m.awaited, _CoroAwait())", + // `yield from` is the same machine over an iterator rather than an + // awaitable, and an exhausted one returns `None` rather than nothing + "list(m.delegated(iter([1, 2, 3])))", + "list(m.delegated(_counting(2)))", + "list(m.delegated(iter([])))", + "_capture(list, m.delegated(_RaiseIter(_Sub(5))))", + "_sent(m.delegated(_counting(2)), (7, 8))", + ], + ); +} + +#[test] +fn a_value_live_across_a_suspension_agrees() { + // python evaluates left to right, so `total + await step(i)` has the read of + // `total` on the stack while the `await` suspends. every shape here holds + // something across a suspension that has no name of its own, and `agree` is what + // says they compile rather than falling back + agree( + "parked", + "\ +async def stepped(i: int) -> int: + return (i * 7) % 13 + +async def summed(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await stepped(i) + i = i + 1 + return total + +async def paired(n: int) -> int: + return (await stepped(n)) + (await stepped(n + 1)) + +async def tripled(n: int) -> int: + return (await stepped(n)) * 100 + (await stepped(n + 1)) * 10 + (await stepped(n + 2)) + +def mixed(a: int, b: int, c: int) -> int: + return a * 100 + b * 10 + c + +async def spanning(n: int) -> int: + return mixed(n * 5, await stepped(n), await stepped(n + 1)) + +async def awaited(step: object, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await step(i) + i = i + 1 + return total + +def echoing(n: int) -> object: + total = 0 + i = 0 + while i < n: + total = total + (yield i) + i = i + 1 + return total + +def twinned() -> object: + return (yield 1) + (yield 2) + +def straddling(n: int) -> object: + yield mixed(n * 5, (yield 1), (yield 2)) + +def recovering(n: int) -> object: + total = n * 2 + try: + yield total + except ValueError: + yield total + 1 + yield total + 2 + +def crossed(xs: list[int], ys: list[int]) -> object: + for a in xs: + base = a * 100 + for b in ys: + yield base + (yield b) + +def counting(n: int) -> object: + yield n + return n * 100 + +def relayed(n: int) -> object: + base = n + 1 + got = yield from counting(n) + yield base + got +", + &[ + "__import__('asyncio').run(m.summed(40))", + "__import__('asyncio').run(m.paired(3))", + "__import__('asyncio').run(m.tripled(3))", + "__import__('asyncio').run(m.spanning(3))", + // `n * 5` is read before the first suspension and used after the *second*, + // so nothing between the two reads it. the static flow calls it dead at the + // first — a suspension is a `return` and goes nowhere — and one field has to + // carry it across both + "_sent(m.straddling(3), (4, 5))", + // arbitrary precision survives the park, which a machine word would not + "__import__('asyncio').run(m.summed(3)) + 10 ** 30", + // awaiting something that *does* suspend, so the loop resumes the frame + "__import__('asyncio').run(m.awaited(_slow, 4))", + "_sent(m.echoing(4), (10, 20, 30, 40))", + "_sent(m.twinned(), (5, 7))", + "list(m.recovering(3))", + // a `throw` the handler catches, and the parked value read after it + "_recovered(m.recovering(3), ValueError, 2)", + "_sent(m.crossed([1, 2], [7, 8]), (100, 200, 300, 400, 500, 600, 700, 800))", + "list(m.relayed(4))", + // a coroutine driven by hand, with no loop underneath it + "[(c := m.summed(3), _capture(c.send, None).value)[1:]]", + ], + ); +} + +#[test] +fn a_handled_throw_leaves_the_generator_usable() { + // the field carrying what `throw` wants raised starts null, and is *emptied* by + // writing `None` into it because no operation stores a null. reading it back as a + // second exception raised `SystemError` at every later resumption + agree( + "rethrown", + "\ +def recovering(n: int) -> object: + try: + yield n + except ValueError: + pass + yield n + 2 + yield n + 3 +", + &[ + "_recovered(m.recovering(3), ValueError, 2)", + "list(m.recovering(3))", + ], + ); +} + +#[test] +fn a_parked_value_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_parkleak"); + let _ = std::fs::remove_dir_all(&dir); + // `label + (yield i)` holds the label across the suspension, so the state object + // owns it for as long as the frame is parked + let source = "\ +def tagged(label: str, times: int) -> object: + i = 0 + while i < times: + yield label + (yield i) + i = i + 1 + +def handling(label: str) -> object: + try: + yield label + except ValueError: + yield label + \"!\" + yield label + \"?\" +"; + if build_source( + source, + "by_diff_parkleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // driven to exhaustion, and abandoned *while suspended* with the value parked + let out = run( + &python, + &dir, + "import gc, sys, by_diff_parkleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 g = m.tagged(label, 2)\n\ + \x20 next(g)\n\ + \x20 g.send('a')\n\ + \x20 g.close()\n\ + del g\n\ + gc.collect()\n\ + print('driven', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.tagged(label, 9)\n\ + \x20 next(g)\n\ + \x20 g.send('a')\n\ + del g\n\ + gc.collect()\n\ + print('parked', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.handling(label)\n\ + \x20 next(g)\n\ + \x20 g.throw(ValueError('boom'))\n\ + del g\n\ + gc.collect()\n\ + print('handled', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + ); + // a suspension inside `except` parks the exception the handler took over from, so + // the state object is holding one when it is dropped mid-handler + assert_eq!(out, "driven stable\nparked stable\nhandled stable"); +} + +#[test] +fn a_coroutine_is_awaitable_and_not_iterable() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_coro"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +async def plain(n: int) -> int: + return n * 2 +"; + if build_source( + source, + "by_diff_coro", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // `asyncio.iscoroutine` tests the abc, and iterating a coroutine has to be an + // error rather than quietly working + let out = run( + &python, + &dir, + "import asyncio, collections.abc as abc, by_diff_coro as m\n\ + c = m.plain(2)\n\ + print(asyncio.iscoroutine(c), isinstance(c, abc.Coroutine))\n\ + print(hasattr(c, 'send'), hasattr(c, 'throw'), hasattr(c, 'close'))\n\ + try:\n list(c)\n\ + except TypeError:\n print('not iterable')\n\ + print(asyncio.run(c))\n", + ); + assert_eq!(out, "True True\nTrue True True\nnot iterable\n4"); +} + +#[test] +fn a_coroutine_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_coroleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +async def echo(label: str) -> str: + return label +"; + if build_source( + source, + "by_diff_coroleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // including a coroutine that is created and never awaited + let out = run( + &python, + &dir, + "import asyncio, gc, sys, warnings, by_diff_coroleak as m\n\ + warnings.simplefilter('ignore')\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(5000):\n\ + \x20 asyncio.run(m.echo(label))\n\ + print('awaited', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(5000):\n\ + \x20 c = m.echo(label)\n\ + del c\n\ + gc.collect()\n\ + print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + ); + assert_eq!(out, "awaited stable\nabandoned stable"); +} + +#[test] +fn with_blocks_agree() { + agree_with_declines( + "withblocks", + "\ +def counted(path: str) -> int: + total = 0 + with open(path) as f: + for line in f: + total = total + 1 + return total + +def guarded(mgr: object) -> str: + with mgr: + return \"body\" + return \"after\" + +def nested(mgr: object, other: object) -> str: + with mgr: + with other: + return \"inner\" + return \"after\" + +def suppressing(mgr: object) -> str: + with mgr: + raise ValueError(\"boom\") + return \"suppressed\" + +def propagating(mgr: object) -> str: + with mgr: + raise ValueError(\"boom\") + return \"not reached\" + +def entered(mgr: object) -> object: + with mgr as value: + return value + return None +", + &[ + // `__exit__` runs on the normal path with `(None, None, None)`, and a + // `return` from inside the body is one of those paths + "_run_recording(m)", + "m.suppressing(_Swallow())", + "[type(e).__name__ for e in [_capture(m.propagating, _Pass())]]", + "m.entered(_Value(7))", + "_run_nested(m)", + ], + ); +} + +#[test] +fn an_early_exit_runs_the_finally_it_is_leaving() { + // this was a silent wrong answer in a shipped feature: a `return` or a `break` + // inside `try` skipped the `finally` entirely + agree( + "unwind", + "\ +def early(log: list[str]) -> str: + try: + return \"body\" + finally: + log.append(\"finally\") + +def looped(log: list[str], n: int) -> str: + i = 0 + while i < n: + try: + if i == 1: + break + finally: + log.append(\"f\") + i = i + 1 + return \"done\" + +def continued(log: list[str], n: int) -> int: + total = 0 + i = 0 + while i < n: + i = i + 1 + try: + if i == 2: + continue + total = total + i + finally: + log.append(\"f\") + return total + +def layered(log: list[str]) -> str: + try: + try: + return \"deep\" + finally: + log.append(\"inner\") + finally: + log.append(\"outer\") +", + &[ + "[(log := [], m.early(log), log)[1:]]", + "[(log := [], m.looped(log, 3), log)[1:]]", + "[(log := [], m.continued(log, 4), log)[1:]]", + // innermost first, and both of them + "[(log := [], m.layered(log), log)[1:]]", + ], + ); +} + +#[test] +fn parameter_defaults_agree() { + agree_with_declines( + "defaults", + "\ +def greet(name: str, greeting: str = \"hi\", times: int = 1) -> str: + return (greeting + \" \" + name) * times + +def offset(a: int, b: int = 10) -> int: + return a + b + +def flagged(a: int, on: bool = True) -> int: + if on: + return a + return -a + +def boxed_none(a: object = None) -> object: + return a + +def boxed_int(a: object = 7) -> object: + return a + +def boxed_bool(a: object = True) -> object: + return a + +def boxed_float(a: object = 1.5) -> object: + return a + +def optional(a: int, extra: object = None) -> object: + if extra is None: + return a + return extra + +def computed(a: int, b: object = []) -> object: + return b +", + &[ + "m.greet('a')", + "m.greet('a', 'yo')", + "m.greet('a', 'yo', 2)", + "m.offset(1)", + "m.offset(1, 2)", + "(m.flagged(3), m.flagged(3, False))", + "(m.optional(1), m.optional(1, 'x'))", + // too few arguments still raises, and so does too many + "[type(e).__name__ for e in [_capture(m.offset)]]", + "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", + // a computed default declines, and the interpreted twin keeps the identity + "m.computed(1)", + // an immediate written into an *object* place has to be boxed: the + // unboxed `None` is a bare byte, and `By_NewRef` of one is a NULL the + // error check reads as a failure with no exception behind it + "m.boxed_none()", + "m.boxed_none(1)", + "m.boxed_int()", + "m.boxed_bool()", + "m.boxed_float()", + "(m.boxed_none() is None, m.boxed_bool() is True)", + ], ); - assert_eq!(out, "abandoned 0\nexhausted 0\nclosed 0\nin handler 0"); } -/// a `throw` raises at the suspension point, and whether a handler catches it or -/// it comes back out, nothing may hold a reference afterwards #[test] -fn throwing_into_a_generator_releases_the_exception() { - let Some((python, dir)) = leak_module("by_diff_genthrow") else { - return; - }; - // two instruments over the same shapes: one instance thrown again and again, - // whose own reference count moves even though the object stays reachable, and a - // fresh instance per call, which a retained reference keeps alive as a countable - // object. - // - // the exception is a global rather than a parameter on purpose. a caught - // exception's traceback holds the frames it passed through, so a local naming it - // is a reference the *test* keeps — 50 of them, in both builds alike - let out = run( - &python, - &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genthrow as m\n\ - def caught():\n\ - \x20 g = m.guarded(9)\n\ - \x20 next(g)\n\ - \x20 g.throw(thrown)\n\ - def uncaught():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - \x20 try:\n\ - \x20 g.throw(thrown)\n\ - \x20 except m.Boom:\n\ - \x20 pass\n\ - def in_handler():\n\ - \x20 g = m.in_handler(9)\n\ - \x20 next(g)\n\ - def fresh(once):\n\ - \x20 def run():\n\ - \x20 global thrown\n\ - \x20 thrown = m.Boom()\n\ - \x20 once()\n\ - \x20 thrown = None\n\ - \x20 return run\n\ - thrown = m.Boom()\n\ - boom = thrown\n\ - gc.collect()\n\ - held = sys.getrefcount(boom)\n\ - for _ in range(50):\n\ - \x20 caught()\n\ - \x20 uncaught()\n\ - gc.collect()\n\ - print('thrown refs', sys.getrefcount(boom) - held)\n\ - print('caught', leaked(m.Boom, fresh(caught)))\n\ - print('uncaught', leaked(m.Boom, fresh(uncaught)))\n\ - print('raised inside', leaked(m.Boom, in_handler))\n" - ), +fn a_default_filled_at_a_native_call_site_agrees() { + // `parameter_defaults_agree` calls these from python, which fills a default in the + // *wrapper* — where the value is already an object. a compiled caller fills it + // inline instead, and that path pushed the default with no coercion at all: a bare + // `length=0` reaching an unannotated parameter put a tagged integer where the + // callee declares a `PyObject *`. it was `shutil.py` failing to build that found it, + // and only because the two representations differ enough for a C compiler to object + agree( + "native_defaults", + "\ +def taking(a, b, length=0, name=\"n\", scale=1.5, on=True, extra=None): + return (a, b, length, name, scale, on, extra) + + +def calling(x: int) -> object: + return taking(x, x) + + +def partly(x: int) -> object: + return taking(x, x, 3) + + +def by_keyword(x: int) -> object: + return taking(x, x, on=False) + + +def annotated(a: int, step: int = 2) -> int: + return a + step + + +def calling_annotated(x: int) -> int: + return annotated(x) +", + &[ + "m.calling(1)", + "m.partly(2)", + "m.by_keyword(3)", + "[m.calling_annotated(n) for n in (0, 5)]", + "m.taking(1, 2)", + "(m.calling(1)[3], type(m.calling(1)[4]).__name__, m.calling(1)[6] is None)", + ], ); - assert_eq!(out, "thrown refs 0\ncaught 0\nuncaught 0\nraised inside 0"); } -/// the same rule outside a generator, which is where it is stated: a re-raise -/// hands the exception to the interpreter and keeps nothing #[test] -fn a_re_raise_does_not_retain_the_exception() { +fn a_string_default_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_reraiseleak"); + let dir = std::env::temp_dir().join("by_diff_defaultleak"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Boom(Exception): - pass - -def bare(x: int) -> int: - try: - raise Boom() - except Boom: - raise - -def unmatched(x: int) -> int: - try: - raise Boom() - except ValueError: - return 1 - return 2 - -def through_finally(x: int) -> int: - try: - raise Boom() - finally: - x = x + 1 +def padded(a: str, fill: str = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\") -> str: + return a + fill "; - let options = Options { - require_native: true, - ..Options::default() - }; - if build_source(source, "by_diff_reraiseleak", &toolchain, &dir, &options).is_err() { + if build_source( + source, + "by_diff_defaultleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { eprintln!("skipping: no working C toolchain"); return; } + // the wrapper releases its arguments, so a default handed over borrowed would be + // released twice — and one handed over with an extra reference would leak let out = run( &python, &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_reraiseleak as m\n\ - def raising(fn):\n\ - \x20 def once():\n\ - \x20 try:\n\ - \x20 fn(1)\n\ - \x20 except m.Boom:\n\ - \x20 pass\n\ - \x20 return once\n\ - for name in ('bare', 'unmatched', 'through_finally'):\n\ - \x20 print(name, leaked(m.Boom, raising(getattr(m, name))))\n" - ), + "import sys, by_diff_defaultleak as m\n\ + fill = 'x' * 40\n\ + before = sys.getrefcount(fill)\n\ + for _ in range(20000):\n\ + \x20 m.padded('a')\n\ + print('stable' if sys.getrefcount(fill) == before else 'moved')\n\ + print(m.padded('a')[:3])\n", ); - assert_eq!(out, "bare 0\nunmatched 0\nthrough_finally 0"); + assert_eq!(out, "stable\naxx"); } #[test] -fn delegation_agrees() { +fn keyword_arguments_agree() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } agree_with_declines( - "delegation", + "keywords", "\ -def inner(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 - return n * 100 +def offset(a: int, b: int = 10) -> int: + return a + b + +def described(name: str, sep: str = \"-\", times: int = 1) -> str: + return (name + sep) * times + +def caller(a: int) -> int: + return offset(a, b=100) + offset(b=1, a=a) + offset(a) + +data class Point: + x: int + y: int + + def shifted(self, dx: int = 0, dy: int = 0) -> int: + return self.x + dx + self.y + dy +", + &[ + // a python caller, by name and by position + "m.offset(1, b=2)", + "m.offset(b=2, a=1)", + "m.offset(1)", + "m.described('a', times=3)", + "m.described(name='a', sep='+', times=2)", + // a compiled caller resolves the names against the callee's signature + "m.caller(1)", + // and a method's keywords work through the type object + "m.Point(1, 2).shifted(dy=10)", + "m.Point(1, 2).shifted(3, dy=10)", + // the error cases match + "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", + "[str(_capture_kw(m.offset, (1,), {'a': 2}))]", + "[str(_capture_kw(m.offset, (), {'zzz': 1}))]", + "[str(_capture(m.offset))]", + ], + ); +} + +#[test] +fn variadic_parameters_agree() { + agree_with_declines( + "variadic", + "\ +def total(*values: int) -> int: + out = 0 + for v in values: + out = out + v + return out + +def named(prefix: str, *rest: str) -> str: + out = prefix + for r in rest: + out = out + r + return out + +def options(a: int, **rest: object) -> int: + return a + len(rest) + +def both(a: int, *rest: int, **named: object) -> int: + return a + len(rest) + len(named) + +def tupled(*values: int) -> object: + return values + +def mapped(**named: object) -> object: + return named -def outer(n: int) -> object: - got = yield from inner(n) - yield got +def calls_total(a: int, b: int) -> int: + return total(a, b, 3) -def chained(xs: list[int]) -> object: - yield from xs - yield from xs +def calls_options(a: int) -> int: + return options(a, x=1, y=2) -def nested(n: int) -> object: - yield from outer(n) +def calls_both(a: int) -> int: + return both(a, 1, 2, k=3, j=4) + +def calls_none() -> int: + return total() ", &[ - "list(m.inner(3))", - // the delegated `return` value is what the expression evaluates to - "list(m.outer(3))", - "list(m.chained([1, 2]))", - "list(m.nested(2))", - "list(m.chained([]))", - "sum(m.outer(4))", - // and the inner generator's own `StopIteration` value reaches the consumer - "[(g := m.inner(1), next(g), _capture(next, g).value)[2:]]", + "m.total()", + "m.total(1, 2, 3)", + "(m.named('a'), m.named('a', 'b', 'c'))", + "(m.options(1), m.options(1, x=2, y=3))", + "m.both(1, 2, 3, k=4)", + // the body sees a real tuple and a real dict + "m.tupled(1, 2)", + "sorted(m.mapped(b=2, a=1).items())", + "type(m.tupled()).__name__", + "type(m.mapped()).__name__", + // a keyword that names no parameter goes to `**kwargs`, or raises + "[str(_capture_kw(m.named, ('a',), {'zzz': 1}))]", + "[str(_capture(m.options))]", + "[m.total(*a) for a in ([1], [1, 2])]", + // and a *compiled* caller packs the tuple and the dict itself + "m.calls_total(1, 2)", + "m.calls_options(5)", + "m.calls_both(1)", + "m.calls_none()", ], ); } #[test] -fn async_await_agrees() { - agree_with_declines( - "asyncawait", - "\ -async def plain(n: int) -> int: - return n * 2 - -async def chained(n: int) -> int: - a = await plain(n) - b = await plain(a) - return b + 1 - -async def nothing() -> None: - pass +fn a_variadic_argument_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_varleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def joined(*parts: str) -> int: + total = 0 + for p in parts: + total = total + len(p) + return total -async def forwards(n: int) -> int: - return await chained(n) -", - &[ - "__import__('asyncio').run(m.plain(5))", - "__import__('asyncio').run(m.chained(3))", - "__import__('asyncio').run(m.nothing())", - "__import__('asyncio').run(m.forwards(1))", - "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", - ], +def mapped(**named: object) -> int: + return len(named) +"; + if build_source( + source, + "by_diff_varleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the wrapper *builds* the tuple and the dict, so it owns them and has to release + // them — and the elements it put in are borrowed from the caller + let out = run( + &python, + &dir, + "import gc, sys, by_diff_varleak as m\n\ + part = 'x' * 40\n\ + before = sys.getrefcount(part)\n\ + for _ in range(20000):\n\ + \x20 m.joined(part, part)\n\ + \x20 m.mapped(a=part)\n\ + print('refs', 'stable' if sys.getrefcount(part) == before else 'leaked')\n\ + gc.collect()\n\ + objects = len(gc.get_objects())\n\ + for _ in range(5000):\n\ + \x20 m.joined(part)\n\ + \x20 m.mapped(a=part, b=part)\n\ + gc.collect()\n\ + print('objects', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", ); + assert_eq!(out, "refs stable\nobjects stable"); } #[test] -fn the_await_protocol_agrees() { - // an `await` reaches its object through `__await__` and drives what comes back - // by sending into it. both halves have edges worth pinning: what counts as - // awaitable at all, and what a finished delegation's value *is* - agree( - "awaitproto", +fn a_decorated_method_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree_with_declines( + "methoddeco", "\ -async def awaited(x: object) -> object: - return await x - -async def twice(x: object, y: object) -> object: - a = await x - b = await y - return (a, b) - -async def guarded(x: object) -> object: - try: - return await x - except ValueError as e: - return 'caught ' + str(e) - -def delegated(xs: object) -> object: - got = yield from xs - yield got +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper -def returns(v: object) -> object: - return v - yield +data class Point: + x: int + y: int -async def returns_async(v: object) -> object: - return v + @property + def total(self) -> int: + return self.x + self.y -def redelegated(v: object) -> object: - got = yield from returns(v) - yield got + @doubling + def raw(self) -> int: + return self.x ", &[ - // the *other* half of the same rule: what a compiled frame's own return - // value becomes on the way out. `StopIteration` reads its argument as an - // argument *list*, so a tuple would arrive spread and an exception - // instance would be raised in place of the `StopIteration` itself - "[_value(m.returns(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2], 'ab')]", - "[_run(m.returns_async(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2])]", - "[list(m.redelegated(v)) for v in ((1, 2), (), (1,), 5, None)]", - "[_run(m.awaited(m.returns_async(v))) for v in ((1, 2), (), 5, None)]", - "_value(m.returns(ValueError('x')))", - "_value(m.returns(StopIteration(9)))", - "_run(m.returns_async(StopIteration(9)))", - // and the exception it rides on is shaped the way python shapes it - "[(lambda e: (repr(e), e.args))(_capture(next, m.returns(v))) \ - for v in ((1, 2), (), 5, None)]", - // an await that completes without ever suspending — the common case - "_run(m.awaited(_Ready(7)))", - "_run(m.twice(_Ready(1), _Ready(2)))", - // one that really suspends, driven by a loop that has to resume it - "_run(m.awaited(_sleeps(9)))", - "_run(m.awaited(_Suspends(11)))", - "_run(m.twice(_Ready(1), _sleeps(2)))", - "_run(m.twice(_Suspends(1), _Suspends(2)))", - // a bare `StopIteration` carries `None`, and a subclass carries its own - "_run(m.awaited(_Raises(StopIteration())))", - "_run(m.awaited(_Raises(StopIteration(3))))", - "_run(m.awaited(_Raises(_Sub(5))))", - // python reads the exception's field, not its `value` attribute - "_run(m.awaited(_Raises(_Shadowed(7))))", - "_Shadowed(7).value", - // a raised *type* rather than an instance still has to become one - "_run(m.awaited(_Raises(StopIteration)))", - // normalising a tuple argument would take only its first element - "_run(m.awaited(_Raises(StopIteration((1, 2)))))", - // anything else escaping the awaited object is not a return value - "_capture_async(m.awaited, _throws(ValueError))", - "_capture_async(m.awaited, _throws(KeyError))", - "_run(m.guarded(_throws(ValueError)))", - "_capture_async(m.guarded, _throws(KeyError))", - // and what is not awaitable at all says so, about the right object - "_capture_async(m.awaited, 5)", - "_capture_async(m.awaited, _counting(1))", - "_capture_async(m.awaited, _NotIter())", - "_capture_async(m.awaited, _CoroAwait())", - // `yield from` is the same machine over an iterator rather than an - // awaitable, and an exhausted one returns `None` rather than nothing - "list(m.delegated(iter([1, 2, 3])))", - "list(m.delegated(_counting(2)))", - "list(m.delegated(iter([])))", - "_capture(list, m.delegated(_RaiseIter(_Sub(5))))", - "_sent(m.delegated(_counting(2)), (7, 8))", + // a property is a descriptor on the type, reached without a call + "m.Point(3, 4).total", + "type(m.Point.total).__name__", + // and a user decorator wraps the native method + "m.Point(3, 4).raw()", ], ); } +/// a decorator that *mutates* what it is handed, next to one that wraps +/// +/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the +/// same object back, so it is the whole class of decorator a compiled method has to +/// stay writable for — a method descriptor takes no attributes at all. and the two +/// class constructions have to be covered separately: `Plain` is built from a spec, so +/// the decorators reach methods this module compiled, while `Shape`'s metaclass rules +/// a spec out and its construction falls back to the interpreted definition, which +/// already carries them #[test] -fn a_value_live_across_a_suspension_agrees() { - // python evaluates left to right, so `total + await step(i)` has the read of - // `total` on the stack while the `await` suspends. every shape here holds - // something across a suspension that has no name of its own, and `agree` is what - // says they compile rather than falling back - agree( - "parked", +fn a_mutating_method_decorator_agrees() { + agree_python( + "mutatingdeco", "\ -async def stepped(i: int) -> int: - return (i * 7) % 13 - -async def summed(n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + await stepped(i) - i = i + 1 - return total +from abc import ABC, abstractmethod -async def paired(n: int) -> int: - return (await stepped(n)) + (await stepped(n + 1)) -async def tripled(n: int) -> int: - return (await stepped(n)) * 100 + (await stepped(n + 1)) * 10 + (await stepped(n + 2)) +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper -def mixed(a: int, b: int, c: int) -> int: - return a * 100 + b * 10 + c -async def spanning(n: int) -> int: - return mixed(n * 5, await stepped(n), await stepped(n + 1)) +def tagging(fn: object) -> object: + def wrapper(self: object) -> object: + return str(fn(self)) + '!' + return wrapper -async def awaited(step: object, n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + await step(i) - i = i + 1 - return total -def echoing(n: int) -> object: - total = 0 - i = 0 - while i < n: - total = total + (yield i) - i = i + 1 - return total +class Plain: + @abstractmethod + def area(self) -> int: + return 3 -def twinned() -> object: - return (yield 1) + (yield 2) + @doubling + def raw(self) -> int: + return 5 -def straddling(n: int) -> object: - yield mixed(n * 5, (yield 1), (yield 2)) + @tagging + @doubling + def stacked(self) -> int: + return 4 -def recovering(n: int) -> object: - total = n * 2 - try: - yield total - except ValueError: - yield total + 1 - yield total + 2 + @property + def total(self) -> int: + return 6 -def crossed(xs: list[int], ys: list[int]) -> object: - for a in xs: - base = a * 100 - for b in ys: - yield base + (yield b) -def counting(n: int) -> object: - yield n - return n * 100 +class Shape(ABC): + @abstractmethod + def area(self) -> int: + return 7 -def relayed(n: int) -> object: - base = n + 1 - got = yield from counting(n) - yield base + got + @doubling + def raw(self) -> int: + return 11 ", &[ - "__import__('asyncio').run(m.summed(40))", - "__import__('asyncio').run(m.paired(3))", - "__import__('asyncio').run(m.tripled(3))", - "__import__('asyncio').run(m.spanning(3))", - // `n * 5` is read before the first suspension and used after the *second*, - // so nothing between the two reads it. the static flow calls it dead at the - // first — a suspension is a `return` and goes nowhere — and one field has to - // carry it across both - "_sent(m.straddling(3), (4, 5))", - // arbitrary precision survives the park, which a machine word would not - "__import__('asyncio').run(m.summed(3)) + 10 ** 30", - // awaiting something that *does* suspend, so the loop resumes the frame - "__import__('asyncio').run(m.awaited(_slow, 4))", - "_sent(m.echoing(4), (10, 20, 30, 40))", - "_sent(m.twinned(), (5, 7))", - "list(m.recovering(3))", - // a `throw` the handler catches, and the parked value read after it - "_recovered(m.recovering(3), ValueError, 2)", - "_sent(m.crossed([1, 2], [7, 8]), (100, 200, 300, 400, 500, 600, 700, 800))", - "list(m.relayed(4))", - // a coroutine driven by hand, with no loop underneath it - "[(c := m.summed(3), _capture(c.send, None).value)[1:]]", + // the mutating decorator ran, and the method it marked still calls + "m.Plain.area.__isabstractmethod__", + "m.Plain().area()", + // the name a decorator reads off a function is still the method's own + "m.Plain.area.__name__", + // a wrapping decorator wraps once, not once per construction + "m.Plain().raw()", + // and the innermost is applied first, so `tagging` sees the doubled value + "m.Plain().stacked()", + "m.Plain().total", + "type(m.Plain.total).__name__", + // an `ABCMeta` base reads the mark, so the abstract set is the same set + "sorted(m.Shape.__abstractmethods__)", + "m.Shape.area.__isabstractmethod__", + "type('S', (m.Shape,), {'area': lambda self: 13})().raw()", + "type('S', (m.Shape,), {'area': lambda self: 13})().area()", ], ); } #[test] -fn a_handled_throw_leaves_the_generator_usable() { - // the field carrying what `throw` wants raised starts null, and is *emptied* by - // writing `None` into it because no operation stores a null. reading it back as a - // second exception raised `SystemError` at every later resumption - agree( - "rethrown", +fn lambdas_agree() { + agree_with_declines( + "lambdas", "\ -def recovering(n: int) -> object: - try: - yield n - except ValueError: - pass - yield n + 2 - yield n + 3 +def adder(n: int) -> ((int) -> int): + return lambda x: x + n + +def twice(n: int) -> int: + f = lambda x: x * 2 + return f(f(n)) + +def picked(flag: bool) -> ((int) -> int): + if flag: + return lambda x: x + 1 + return lambda x: x - 1 + +def counter() -> (() -> int): + n = 0 + f = lambda: n + n = 1 + return f + +def each(values: list[int]) -> list[object]: + out = [] + for v in values: + out.append(lambda: v) + return out ", &[ - "_recovered(m.recovering(3), ValueError, 2)", - "list(m.recovering(3))", + "m.adder(3)(4)", + "m.twice(5)", + "(m.picked(True)(10), m.picked(False)(10))", + // the lambda closes over the *variable*, so it sees the later write + "m.counter()()", + // and every lambda a loop makes shares one cell, as in python + "[f() for f in m.each([1, 2, 3])]", + "sorted([3, 1, 2], key=m.adder(0))", + "[m.adder(a)(a) for a in (0, -3, 10 ** 20)]", ], ); } #[test] -fn a_parked_value_does_not_leak() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_parkleak"); - let _ = std::fs::remove_dir_all(&dir); - // `label + (yield i)` holds the label across the suspension, so the state object - // owns it for as long as the frame is parked - let source = "\ -def tagged(label: str, times: int) -> object: - i = 0 - while i < times: - yield label + (yield i) - i = i + 1 - -def handling(label: str) -> object: - try: - yield label - except ValueError: - yield label + \"!\" - yield label + \"?\" -"; - if build_source( - source, - "by_diff_parkleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); +fn a_loop_over_native_instances_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); return; } - // driven to exhaustion, and abandoned *while suspended* with the value parked - let out = run( - &python, - &dir, - "import gc, sys, by_diff_parkleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 g = m.tagged(label, 2)\n\ - \x20 next(g)\n\ - \x20 g.send('a')\n\ - \x20 g.close()\n\ - del g\n\ - gc.collect()\n\ - print('driven', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.tagged(label, 9)\n\ - \x20 next(g)\n\ - \x20 g.send('a')\n\ - del g\n\ - gc.collect()\n\ - print('parked', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.handling(label)\n\ - \x20 next(g)\n\ - \x20 g.throw(ValueError('boom'))\n\ - del g\n\ - gc.collect()\n\ - print('handled', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + // narrowing the element to an emitted class is a checked unbox like any other — + // asking the *free* narrowable rather than the one that knows this module's + // layouts declined the whole loop + agree( + "loopnative", + "\ +frozen data class Vec2: + x: float + y: float + +data class Loose: + n: int + +def total(vs: list[Vec2]) -> float: + out = 0.0 + for v in vs: + out = out + v.x * v.x + v.y * v.y + return out + +def summed(xs: list[Loose]) -> int: + return sum([x.n for x in xs]) + +def wrong_element(vs: list[Vec2]) -> float: + out = 0.0 + for v in vs: + out = out + v.x + return out +", + &[ + "round(m.total([m.Vec2(1.0, 2.0), m.Vec2(3.0, 4.0)]), 1)", + "m.summed([m.Loose(1), m.Loose(2)])", + "round(m.total([]), 1)", + // and the narrowing is a real check: a list that lies raises + "[type(e).__name__ for e in [_capture(m.wrong_element, ['not a vec'])]]", + ], ); - // a suspension inside `except` parks the exception the handler took over from, so - // the state object is holding one when it is dropped mid-handler - assert_eq!(out, "driven stable\nparked stable\nhandled stable"); } -#[test] -fn a_coroutine_is_awaitable_and_not_iterable() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_coro"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -async def plain(n: int) -> int: - return n * 2 +/// the process's own peak footprint, which is the only thing that sees a buffer +/// that is not a `PyObject` +/// +/// windows has no `resource` module; `GetProcessMemoryInfo` answers the same +/// question there. the structure is spelled by field *width* rather than by C +/// name because `ctypes.c_ulong` is four bytes on windows and eight elsewhere — +/// a `DWORD` written as `c_ulong` would move every offset behind it — and its +/// size is checked on every platform, so the branch only windows takes is still +/// held to its layout here +const PEAK_FOOTPRINT: &str = "\ +import ctypes + +class _Counters(ctypes.Structure): + _fields_ = [(\"cb\", ctypes.c_uint32), (\"page_faults\", ctypes.c_uint32)] + [ + (name, ctypes.c_size_t) + for name in (\"peak_working_set\", \"working_set\", + \"quota_peak_paged_pool\", \"quota_paged_pool\", + \"quota_peak_non_paged_pool\", \"quota_non_paged_pool\", + \"pagefile\", \"peak_pagefile\")] + +assert ctypes.sizeof(_Counters) == 8 + 8 * ctypes.sizeof(ctypes.c_size_t) + +def _windows_peak(): + kernel32 = ctypes.windll.kernel32 + kernel32.GetCurrentProcess.restype = ctypes.c_void_p + kernel32.K32GetProcessMemoryInfo.restype = ctypes.c_int32 + kernel32.K32GetProcessMemoryInfo.argtypes = ( + ctypes.c_void_p, ctypes.POINTER(_Counters), ctypes.c_uint32) + counters = _Counters() + counters.cb = ctypes.sizeof(counters) + if not kernel32.K32GetProcessMemoryInfo( + kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): + raise ctypes.WinError() + return counters.peak_working_set + +try: + import resource +except ImportError: + _peak = _windows_peak +else: + def _peak(): + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss "; - if build_source( - source, - "by_diff_coro", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // `asyncio.iscoroutine` tests the abc, and iterating a coroutine has to be an - // error rather than quietly working - let out = run( - &python, - &dir, - "import asyncio, collections.abc as abc, by_diff_coro as m\n\ - c = m.plain(2)\n\ - print(asyncio.iscoroutine(c), isinstance(c, abc.Coroutine))\n\ - print(hasattr(c, 'send'), hasattr(c, 'throw'), hasattr(c, 'close'))\n\ - try:\n list(c)\n\ - except TypeError:\n print('not iterable')\n\ - print(asyncio.run(c))\n", - ); - assert_eq!(out, "True True\nTrue True True\nnot iterable\n4"); -} #[test] -fn a_coroutine_does_not_leak() { +fn an_unboxed_array_does_not_leak_its_buffer() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_coroleak"); + let dir = std::env::temp_dir().join("by_diff_arrayleak"); let _ = std::fs::remove_dir_all(&dir); + // the buffer is `PyMem_Malloc`, not a `PyObject` — so a leak of one is invisible + // to `gc.get_objects()` and to a refcount check. the process's own footprint is + // the only thing that sees it let source = "\ -async def echo(label: str) -> str: - return label +def churn(rounds: int) -> float: + out = 0.0 + r = 0 + while r < rounds: + xs = [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] + out = out + xs[0] + r = r + 1 + return out "; if build_source( source, - "by_diff_coroleak", + "by_diff_arrayleak", &toolchain, &dir, &Options::default(), @@ -4683,1843 +6816,1918 @@ async def echo(label: str) -> str: eprintln!("skipping: no working C toolchain"); return; } - // including a coroutine that is created and never awaited let out = run( &python, &dir, - "import asyncio, gc, sys, warnings, by_diff_coroleak as m\n\ - warnings.simplefilter('ignore')\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(5000):\n\ - \x20 asyncio.run(m.echo(label))\n\ - print('awaited', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(5000):\n\ - \x20 c = m.echo(label)\n\ - del c\n\ - gc.collect()\n\ - print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + &format!( + "import by_diff_arrayleak as m\n{PEAK_FOOTPRINT}\ + m.churn(1000)\n\ + before = _peak()\n\ + m.churn(400000)\n\ + after = _peak()\n\ + print(after - before < before // 2 or after - before < 4_000_000)\n" + ), ); - assert_eq!(out, "awaited stable\nabandoned stable"); + assert_eq!(out, "True", "the array buffer leaks"); } #[test] -fn with_blocks_agree() { - agree_with_declines( - "withblocks", +fn container_fast_paths_agree_and_respect_subclasses() { + // the fast paths are guarded on the *exact* type: a subclass may override + // `__getitem__` or `__missing__`, and a fast path that ignored that would be a + // wrong answer rather than a fast one + agree( + "containerprim", "\ -def counted(path: str) -> int: - total = 0 - with open(path) as f: - for line in f: - total = total + 1 - return total - -def guarded(mgr: object) -> str: - with mgr: - return \"body\" - return \"after\" +def indexed(xs: list[int], i: int) -> int: + return xs[i] -def nested(mgr: object, other: object) -> str: - with mgr: - with other: - return \"inner\" - return \"after\" +def written(xs: list[int], i: int, v: int) -> str: + xs[i] = v + return str(xs) -def suppressing(mgr: object) -> str: - with mgr: - raise ValueError(\"boom\") - return \"suppressed\" +def looked_up(d: dict[str, int], k: str) -> int: + return d[k] -def propagating(mgr: object) -> str: - with mgr: - raise ValueError(\"boom\") - return \"not reached\" +def sized(xs: object) -> int: + return len(xs) -def entered(mgr: object) -> object: - with mgr as value: - return value - return None +def tupled(t: tuple[int, int], i: int) -> object: + return t[i] ", &[ - // `__exit__` runs on the normal path with `(None, None, None)`, and a - // `return` from inside the body is one of those paths - "_run_recording(m)", - "m.suppressing(_Swallow())", - "[type(e).__name__ for e in [_capture(m.propagating, _Pass())]]", - "m.entered(_Value(7))", - "_run_nested(m)", + "m.indexed([1, 2, 3], 1)", + "m.indexed([1, 2, 3], -1)", + "m.written([1, 2, 3], 0, 9)", + "m.looked_up({'a': 1}, 'a')", + "m.sized([1, 2, 3])", + "m.sized({'a': 1})", + "m.sized('abc')", + "m.tupled((4, 5), 1)", + // every error is python's own, message and class included + "[(type(e).__name__, str(e)) for e in [_capture(m.indexed, [1], 5)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.written, [1], 5, 0)]]", + "[(type(e).__name__, repr(str(e))) for e in [_capture(m.looked_up, {}, 'zz')]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.tupled, (1,), 9)]]", + // a *subclass* overriding the protocol must still be honoured + "m.indexed(type('L', (list,), {'__getitem__': lambda s, i: 99})([1, 2]), 0)", + "m.looked_up(type('D', (dict,), {'__missing__': lambda s, k: 77})(), 'nope')", + "m.sized(type('L', (list,), {'__len__': lambda s: 42})([1]))", ], ); } #[test] -fn an_early_exit_runs_the_finally_it_is_leaving() { - // this was a silent wrong answer in a shipped feature: a `return` or a `break` - // inside `try` skipped the `finally` entirely - agree( - "unwind", +fn a_comprehension_over_range_agrees() { + // the statement form has had a counting loop all along; the comprehension drove + // the iteration protocol instead — a `range` object, an iterator, a `next` and + // an unbox *per element*, for a loop whose bounds are right there + agree_with_declines( + "comprange", "\ -def early(log: list[str]) -> str: - try: - return \"body\" - finally: - log.append(\"finally\") +def squares(n: int) -> object: + return [i * i for i in range(n)] -def looped(log: list[str], n: int) -> str: - i = 0 - while i < n: - try: - if i == 1: - break - finally: - log.append(\"f\") - i = i + 1 - return \"done\" +def filtered(n: int) -> object: + return [i for i in range(n) if i % 3 == 0] -def continued(log: list[str], n: int) -> int: - total = 0 - i = 0 - while i < n: - i = i + 1 - try: - if i == 2: - continue - total = total + i - finally: - log.append(\"f\") - return total +def from_two(n: int) -> object: + return [i for i in range(2, n)] -def layered(log: list[str]) -> str: - try: - try: - return \"deep\" - finally: - log.append(\"inner\") - finally: - log.append(\"outer\") +def nested(n: int) -> object: + return [i * j for i in range(n) for j in range(n)] + +def buffered(n: int) -> float: + xs = [i * 1.5 for i in range(n)] + out = 0.0 + for x in xs: + out = out + x + return out + +def empty(n: int) -> float: + xs = [i * 1.5 for i in range(0)] + out = 0.0 + for x in xs: + out = out + x + return out ", &[ - "[(log := [], m.early(log), log)[1:]]", - "[(log := [], m.looped(log, 3), log)[1:]]", - "[(log := [], m.continued(log, 4), log)[1:]]", - // innermost first, and both of them - "[(log := [], m.layered(log), log)[1:]]", + "m.squares(5)", + "m.filtered(10)", + "m.from_two(6)", + "m.nested(3)", + "m.squares(0)", + "m.from_two(1)", + "m.buffered(5)", + "m.empty(0)", + // a negative bound is an empty range, not a backwards one + "m.squares(-3)", ], ); } #[test] -fn parameter_defaults_agree() { +fn a_loop_over_an_unboxed_array_agrees() { + // the shape the whole representation exists for: an `i64` counter, no iterator + // object, no null test per step, and no bounds check — the counter is the + // lowering's own, so it is in range by construction agree_with_declines( - "defaults", + "arrayloop", "\ -def greet(name: str, greeting: str = \"hi\", times: int = 1) -> str: - return (greeting + \" \" + name) * times - -def offset(a: int, b: int = 10) -> int: - return a + b - -def flagged(a: int, on: bool = True) -> int: - if on: - return a - return -a - -def boxed_none(a: object = None) -> object: - return a +def total(n: int) -> float: + xs = [1.5, 2.5, 3.5, 4.5] + out = 0.0 + for x in xs: + out = out + x + return out -def boxed_int(a: object = 7) -> object: - return a +def empty(n: int) -> float: + xs = [1.0] + out = 0.0 + for x in xs: + out = out + x + return out -def boxed_bool(a: object = True) -> object: - return a +def broken(n: int) -> float: + xs = [1.0, 2.0, 3.0] + out = 0.0 + for x in xs: + if x > 1.5: + break + out = out + x + else: + out = -1.0 + return out -def boxed_float(a: object = 1.5) -> object: - return a +def skipped(n: int) -> float: + xs = [1.0, 2.0, 3.0] + out = 0.0 + for x in xs: + if x > 1.5: + continue + out = out + x + return out -def optional(a: int, extra: object = None) -> object: - if extra is None: - return a - return extra +def exhausted(n: int) -> float: + xs = [1.0, 2.0] + out = 0.0 + for x in xs: + out = out + x + else: + out = out + 100.0 + return out -def computed(a: int, b: object = []) -> object: - return b +def flags(n: int) -> int: + bs = [True, False, True, True] + total = 0 + for b in bs: + if b: + total = total + 1 + return total ", &[ - "m.greet('a')", - "m.greet('a', 'yo')", - "m.greet('a', 'yo', 2)", - "m.offset(1)", - "m.offset(1, 2)", - "(m.flagged(3), m.flagged(3, False))", - "(m.optional(1), m.optional(1, 'x'))", - // too few arguments still raises, and so does too many - "[type(e).__name__ for e in [_capture(m.offset)]]", - "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", - // a computed default declines, and the interpreted twin keeps the identity - "m.computed(1)", - // an immediate written into an *object* place has to be boxed: the - // unboxed `None` is a bare byte, and `By_NewRef` of one is a NULL the - // error check reads as a failure with no exception behind it - "m.boxed_none()", - "m.boxed_none(1)", - "m.boxed_int()", - "m.boxed_bool()", - "m.boxed_float()", - "(m.boxed_none() is None, m.boxed_bool() is True)", + "m.total(0)", + "m.empty(0)", + // `break` skips the `else`, `continue` does not + "m.broken(0)", + "m.skipped(0)", + "m.exhausted(0)", + "m.flags(0)", ], ); } #[test] -fn a_default_filled_at_a_native_call_site_agrees() { - // `parameter_defaults_agree` calls these from python, which fills a default in the - // *wrapper* — where the value is already an object. a compiled caller fills it - // inline instead, and that path pushed the default with no coercion at all: a bare - // `length=0` reaching an unannotated parameter put a tagged integer where the - // callee declares a `PyObject *`. it was `shutil.py` failing to build that found it, - // and only because the two representations differ enough for a C compiler to object +fn a_list_that_escapes_keeps_being_a_list() { + // the buffer is an optimization, not a restriction: a name that leaves the + // function never earns one in the first place, so it compiles exactly as it did + // before the representation existed agree( - "native_defaults", + "bufferescape", "\ -def taking(a, b, length=0, name=\"n\", scale=1.5, on=True, extra=None): - return (a, b, length, name, scale, on, extra) - - -def calling(x: int) -> object: - return taking(x, x) - - -def partly(x: int) -> object: - return taking(x, x, 3) - - -def by_keyword(x: int) -> object: - return taking(x, x, on=False) +def returned(n: int) -> object: + xs = [1.0, 2.0] + return xs +def passed(n: int) -> int: + xs = [1.0, 2.0] + return len(sorted(xs)) -def annotated(a: int, step: int = 2) -> int: - return a + step +def stored(n: int) -> object: + xs = [1.0, 2.0] + return [xs, xs] +def kept(n: int) -> float: + xs = [1.0, 2.0] + return xs[0] + xs[1] -def calling_annotated(x: int) -> int: - return annotated(x) +def looped(n: int) -> float: + xs = [1.0, 2.0] + out = 0.0 + for x in xs: + out = out + x + return out ", &[ - "m.calling(1)", - "m.partly(2)", - "m.by_keyword(3)", - "[m.calling_annotated(n) for n in (0, 5)]", - "m.taking(1, 2)", - "(m.calling(1)[3], type(m.calling(1)[4]).__name__, m.calling(1)[6] is None)", + // these keep a real list, and a real list is what comes back + "m.returned(0)", + "type(m.returned(0)).__name__", + "m.passed(0)", + "m.stored(0)", + // and these earn the buffer, invisibly + "m.kept(0)", + "m.looped(0)", ], ); } #[test] -fn a_string_default_does_not_leak() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_defaultleak"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def padded(a: str, fill: str = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\") -> str: - return a + fill -"; - if build_source( - source, - "by_diff_defaultleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the wrapper releases its arguments, so a default handed over borrowed would be - // released twice — and one handed over with an extra reference would leak - let out = run( - &python, - &dir, - "import sys, by_diff_defaultleak as m\n\ - fill = 'x' * 40\n\ - before = sys.getrefcount(fill)\n\ - for _ in range(20000):\n\ - \x20 m.padded('a')\n\ - print('stable' if sys.getrefcount(fill) == before else 'moved')\n\ - print(m.padded('a')[:3])\n", - ); - assert_eq!(out, "stable\naxx"); -} - -#[test] -fn keyword_arguments_agree() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree_with_declines( - "keywords", +fn a_from_import_inside_a_body_agrees() { + // plain python, so the interpreted leg is the source itself: the transpiler's + // lazy-import polyfill resolves `from pkg import submodule` wrongly, which would + // otherwise break the leg this is measured against rather than the compiled one + agree_python( + "fromimport", "\ -def offset(a: int, b: int = 10) -> int: - return a + b +def one(n: int) -> str: + from math import sqrt + return str(round(sqrt(n), 3)) + +def several(s: str) -> str: + from os.path import basename, dirname + return dirname(s) + '|' + basename(s) + +def aliased(n: int) -> str: + from math import sqrt as root + return str(round(root(n), 3)) -def described(name: str, sep: str = \"-\", times: int = 1) -> str: - return (name + sep) * times +# `urllib.parse` is not an attribute of `urllib` until something imports it, so +# this is the fromlist doing its job rather than an attribute read +def submodule(s: str) -> str: + from urllib import parse + return parse.quote(s) -def caller(a: int) -> int: - return offset(a, b=100) + offset(b=1, a=a) + offset(a) +def dotted(s: str) -> str: + from urllib.parse import quote + return quote(s) -data class Point: - x: int - y: int +# a name the module does not have is an `ImportError`, not the `AttributeError` a +# plain attribute read would give — a guarded lazy import rests on the difference +def absent() -> object: + from os import definitely_not_a_real_name + return definitely_not_a_real_name - def shifted(self, dx: int = 0, dy: int = 0) -> int: - return self.x + dx + self.y + dy +def guarded() -> str: + try: + from os import definitely_not_a_real_name + except ImportError: + return 'caught' + return 'no' ", &[ - // a python caller, by name and by position - "m.offset(1, b=2)", - "m.offset(b=2, a=1)", - "m.offset(1)", - "m.described('a', times=3)", - "m.described(name='a', sep='+', times=2)", - // a compiled caller resolves the names against the callee's signature - "m.caller(1)", - // and a method's keywords work through the type object - "m.Point(1, 2).shifted(dy=10)", - "m.Point(1, 2).shifted(3, dy=10)", - // the error cases match - "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", - "[str(_capture_kw(m.offset, (1,), {'a': 2}))]", - "[str(_capture_kw(m.offset, (), {'zzz': 1}))]", - "[str(_capture(m.offset))]", + "m.one(2)", + "m.several('/a/b/c.txt')", + "m.aliased(2)", + "m.submodule('a b/c')", + "m.dotted('a b/c')", + "m.guarded()", + // the type, the message and every attribute an `except` clause reads + "(lambda e: (type(e).__name__, str(e), e.name, e.path, e.name_from))(_capture(m.absent))", + // a thousand imports must not accumulate references to the module. a + // leaked module is one object with a climbing refcount, which no + // object-count check can see + "_repeated(lambda: m.one(2), 1000)", + "_refdelta('math', lambda: m.one(2), 500)", + "_refdelta('urllib.parse', lambda: m.submodule('a b'), 500)", ], ); } #[test] -fn variadic_parameters_agree() { - agree_with_declines( - "variadic", +fn the_remaining_statement_and_expression_forms_agree() { + agree( + "coverage", "\ -def total(*values: int) -> int: - out = 0 - for v in values: - out = out + v - return out +def imports(n: int) -> str: + import math + return str(round(math.sqrt(n), 3)) -def named(prefix: str, *rest: str) -> str: - out = prefix - for r in rest: - out = out + r - return out +def aliased(n: int) -> str: + import os.path as p + return str(p.basename('/a/b')) -def options(a: int, **rest: object) -> int: - return a + len(rest) +def del_key(d: dict[str, int]) -> str: + del d['a'] + return str(sorted(d.items())) -def both(a: int, *rest: int, **named: object) -> int: - return a + len(rest) + len(named) +def del_item(xs: list[int]) -> str: + del xs[0] + return str(xs) -def tupled(*values: int) -> object: - return values +def del_attr(o: object) -> str: + del o.gone + return str(hasattr(o, 'gone')) -def mapped(**named: object) -> object: - return named +def ellipsis(n: int) -> object: + return ... -def calls_total(a: int, b: int) -> int: - return total(a, b, 3) +def negated(s: object) -> object: + return -s -def calls_options(a: int) -> int: - return options(a, x=1, y=2) +def inverted(s: object) -> object: + return ~s -def calls_both(a: int) -> int: - return both(a, 1, 2, k=3, j=4) +def sliced(xs: list[int]) -> str: + return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[:2]) + str(xs[::-1]) -def calls_none() -> int: - return total() +def slice_assigned(xs: list[int]) -> str: + xs[1:3] = [9, 9, 9] + return str(xs) + +def slice_deleted(xs: list[int]) -> str: + del xs[1:3] + return str(xs) + +def walrus(xs: list[int]) -> str: + if (n := len(xs)) > 2: + return 'big ' + str(n) + return 'small ' + str(n) + +def genexp(xs: list[int]) -> str: + return str(sum(x * 2 for x in xs)) + str(any(x > 2 for x in xs)) + str(max(x for x in xs)) + +def declared_global(n: int) -> int: + global _counter + return n ", &[ - "m.total()", - "m.total(1, 2, 3)", - "(m.named('a'), m.named('a', 'b', 'c'))", - "(m.options(1), m.options(1, x=2, y=3))", - "m.both(1, 2, 3, k=4)", - // the body sees a real tuple and a real dict - "m.tupled(1, 2)", - "sorted(m.mapped(b=2, a=1).items())", - "type(m.tupled()).__name__", - "type(m.mapped()).__name__", - // a keyword that names no parameter goes to `**kwargs`, or raises - "[str(_capture_kw(m.named, ('a',), {'zzz': 1}))]", - "[str(_capture(m.options))]", - "[m.total(*a) for a in ([1], [1, 2])]", - // and a *compiled* caller packs the tuple and the dict itself - "m.calls_total(1, 2)", - "m.calls_options(5)", - "m.calls_both(1)", - "m.calls_none()", + "m.imports(16)", + "m.aliased(0)", + "m.del_key({'a': 1, 'b': 2})", + "m.del_item([1, 2, 3])", + "m.ellipsis(0)", + "m.sliced([1, 2, 3, 4])", + "m.slice_assigned([1, 2, 3, 4])", + "m.slice_deleted([1, 2, 3, 4])", + "m.walrus([1, 2, 3])", + "m.walrus([1])", + "m.genexp([1, 2, 3])", + "m.declared_global(4)", + // the protocol forms, on a type that answers them + "m.negated(type('N', (), {'__neg__': lambda s: 'neg'})())", + "m.inverted(type('N', (), {'__invert__': lambda s: 'inv'})())", + "m.del_attr(type('A', (), {})() if False else __import__('types').SimpleNamespace(gone=1))", + // and the errors each raises + "[(type(e).__name__, str(e)) for e in [_capture(m.del_key, {})]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.del_item, [])]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.negated, object())]]", ], ); } #[test] -fn a_variadic_argument_does_not_leak() { +fn a_bad_first_argument_raises_rather_than_crashing() { + // the wrapper releases every argument local on the error path, so one whose + // declaration a `goto` skipped would be released while indeterminate. a wrong + // type in the *first* parameter is the reachable case: it jumps over the rest. + // + // only the exception *type* is compared: the boundary rejects a bad argument + // where the interpreted leg gets as far as the operation that uses it, so the + // two agree that it is a `TypeError` and not on where it was raised + agree( + "badarg", + "\ +def two(a: int, b: str) -> int: + return a + len(b) + +def three(a: int, b: str, c: list[int]) -> int: + return a + len(b) + len(c) +", + &[ + "type(_capture(m.two, 'x', 'y')).__name__", + "type(_capture(m.two, 1, 2)).__name__", + "type(_capture(m.three, 'x', 'y', [1])).__name__", + "type(_capture(m.three, 1, 'y', 'z')).__name__", + "m.two(1, 'yy')", + "m.three(1, 'yy', [1, 2])", + ], + ); +} + +#[test] +fn a_plain_python_loop_closure_shares_its_binding() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_varleak"); - let _ = std::fs::remove_dir_all(&dir); + // python's loop binding is shared by every iteration, so all three closures + // see the last value. basedpython's is per-iteration. the compiled half has to + // follow the *source* language, not a flag — a `.py` fallback is python let source = "\ -def joined(*parts: str) -> int: - total = 0 - for p in parts: - total = total + len(p) - return total - -def mapped(**named: object) -> int: - return len(named) -"; - if build_source( - source, - "by_diff_varleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the wrapper *builds* the tuple and the dict, so it owns them and has to release - // them — and the elements it put in are borrowed from the caller - let out = run( - &python, - &dir, - "import gc, sys, by_diff_varleak as m\n\ - part = 'x' * 40\n\ - before = sys.getrefcount(part)\n\ - for _ in range(20000):\n\ - \x20 m.joined(part, part)\n\ - \x20 m.mapped(a=part)\n\ - print('refs', 'stable' if sys.getrefcount(part) == before else 'leaked')\n\ - gc.collect()\n\ - objects = len(gc.get_objects())\n\ - for _ in range(5000):\n\ - \x20 m.joined(part)\n\ - \x20 m.mapped(a=part, b=part)\n\ - gc.collect()\n\ - print('objects', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", - ); - assert_eq!(out, "refs stable\nobjects stable"); +def counters() -> list[object]: + out = [] + for i in range(3): + def get() -> int: + return i + out.append(get) + return [f() for f in out] +"; + let dir = std::env::temp_dir().join("by_diff_pyloop"); + let interpreted = std::env::temp_dir().join("by_diff_pyloop_i"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&interpreted); + std::fs::create_dir_all(&interpreted).expect("the directory is created"); + std::fs::write(interpreted.join("by_diff_pyloop.py"), source).expect("written"); + + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + let built = match build_source(source, "by_diff_pyloop", &toolchain, &dir, &options) { + Ok(built) => built, + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); + + let body = "import by_diff_pyloop as m\nprint(m.counters())\n"; + let compiled = run(&python, &dir, body); + assert_eq!(compiled, run(&python, &interpreted, body)); + assert!(compiled.contains("[2, 2, 2]"), "{compiled}"); } #[test] -fn a_decorated_method_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree_with_declines( - "methoddeco", +fn dunder_methods_fill_their_type_slots() { + // a method table cannot fill a slot: `repr(x)` reads `tp_repr` and never looks + // the name up. so each of these has to work *both* ways — through the slot and + // as an ordinary method + agree_python( + "dunders", "\ -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +class Bag: + def __init__(self, items: list[int]) -> None: + self.items = items -data class Point: - x: int - y: int + def __repr__(self) -> str: + return 'Bag(' + str(self.items) + ')' - @property - def total(self) -> int: - return self.x + self.y + def __str__(self) -> str: + return 'bag of ' + str(len(self.items)) - @doubling - def raw(self) -> int: - return self.x + def __len__(self) -> int: + return len(self.items) + + def __bool__(self) -> bool: + return len(self.items) > 1 ", &[ - // a property is a descriptor on the type, reached without a call - "m.Point(3, 4).total", - "type(m.Point.total).__name__", - // and a user decorator wraps the native method - "m.Point(3, 4).raw()", + "repr(m.Bag([1, 2, 3]))", + "str(m.Bag([1, 2, 3]))", + "len(m.Bag([1, 2, 3]))", + "bool(m.Bag([1, 2, 3]))", + "bool(m.Bag([1]))", + "len(m.Bag([]))", + "bool(m.Bag([]))", + "f'{m.Bag([1])}'", + "'{}'.format(m.Bag([1, 2]))", + "m.Bag([1, 2]).__repr__()", + "m.Bag([1, 2]).__len__()", + "[b for b in [m.Bag([]), m.Bag([1, 2])] if b]", ], ); } -/// a decorator that *mutates* what it is handed, next to one that wraps -/// -/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the -/// same object back, so it is the whole class of decorator a compiled method has to -/// stay writable for — a method descriptor takes no attributes at all. and the two -/// class constructions have to be covered separately: `Plain` is built from a spec, so -/// the decorators reach methods this module compiled, while `Shape`'s metaclass rules -/// a spec out and its construction falls back to the interpreted definition, which -/// already carries them #[test] -fn a_mutating_method_decorator_agrees() { +fn calling_a_resumable_frame_hands_back_the_state_object() { + // whatever the annotation says the body produces, the *call* gives the state + // object and the iteration or the `await` turns it back. there are three + // signature maps — module-level, methods, nested — and this has to hold in all of + // them, or the caller assigns a `PyObject *` into the annotation's representation agree_python( - "mutatingdeco", + "resumablecall", "\ -from abc import ABC, abstractmethod - +from typing import Any -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +def flags(n): + i = 0 + while i < n: + yield i > 0 + i = i + 1 -def tagging(fn: object) -> object: - def wrapper(self: object) -> object: - return str(fn(self)) + '!' - return wrapper +async def doubled(n: int) -> int: + return n * 2 -class Plain: - @abstractmethod - def area(self) -> int: - return 3 - @doubling - def raw(self) -> int: - return 5 +class Holder: + def __init__(self, n: int) -> None: + self.n = n - @tagging - @doubling - def stacked(self) -> int: - return 4 + def counted(self): + i = 0 + while i < self.n: + yield i == 0 + i = i + 1 - @property def total(self) -> int: - return 6 + out = 0 + for seen in self.counted(): + if seen: + out = out + 1 + return out -class Shape(ABC): - @abstractmethod - def area(self) -> int: - return 7 +def nested(n: int) -> int: + def inner(): + i = 0 + while i < n: + yield i < 1 + i = i + 1 + out = 0 + for seen in inner(): + if seen: + out = out + 1 + return out - @doubling - def raw(self) -> int: - return 11 + +def drained(n: int) -> int: + out = 0 + for seen in flags(n): + if seen: + out = out + 1 + return out ", &[ - // the mutating decorator ran, and the method it marked still calls - "m.Plain.area.__isabstractmethod__", - "m.Plain().area()", - // the name a decorator reads off a function is still the method's own - "m.Plain.area.__name__", - // a wrapping decorator wraps once, not once per construction - "m.Plain().raw()", - // and the innermost is applied first, so `tagging` sees the doubled value - "m.Plain().stacked()", - "m.Plain().total", - "type(m.Plain.total).__name__", - // an `ABCMeta` base reads the mark, so the abstract set is the same set - "sorted(m.Shape.__abstractmethods__)", - "m.Shape.area.__isabstractmethod__", - "type('S', (m.Shape,), {'area': lambda self: 13})().raw()", - "type('S', (m.Shape,), {'area': lambda self: 13})().area()", + "list(m.flags(3))", + "m.drained(3)", + "m.drained(0)", + "_run(m.doubled(5))", + "m.Holder(4).total()", + "list(m.Holder(2).counted())", + "m.nested(3)", + "m.nested(0)", ], ); } #[test] -fn lambdas_agree() { - agree_with_declines( - "lambdas", +fn a_class_of_only_methods_compiles() { + // no `__init__` is not the same as no *layout*: a class of methods has an empty + // one, which is as representable as any other. `object.__init__` is what rejects a + // call with arguments, and it names the class rather than a method it does not have + // + // `Empty` is where the class is a *static* type, which nothing can build on, so it + // fills the pair of slots itself and raises through them. `Consts` is a base, so it + // is a heap type and leaves both to `object` — where the message carries the module, + // because a type built from a spec keeps its module in `tp_name`. that difference is + // the price of not publishing an `__init__` the source never wrote: one installed + // anyway stands between everything built on the class and the `object.__init__` it + // should have reached + agree_python( + "onlymethods", "\ -def adder(n: int) -> ((int) -> int): - return lambda x: x + n +class Helpers: + def doubled(self, n: int) -> int: + return n * 2 -def twice(n: int) -> int: - f = lambda x: x * 2 - return f(f(n)) + def summed(self, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + i + i = i + 1 + return total -def picked(flag: bool) -> ((int) -> int): - if flag: - return lambda x: x + 1 - return lambda x: x - 1 -def counter() -> (() -> int): - n = 0 - f = lambda: n - n = 1 - return f +class Consts: + SCALE = 3 -def each(values: list[int]) -> list[object]: - out = [] - for v in values: - out.append(lambda: v) - return out -", - &[ - "m.adder(3)(4)", - "m.twice(5)", - "(m.picked(True)(10), m.picked(False)(10))", - // the lambda closes over the *variable*, so it sees the later write - "m.counter()()", - // and every lambda a loop makes shares one cell, as in python - "[f() for f in m.each([1, 2, 3])]", - "sorted([3, 1, 2], key=m.adder(0))", - "[m.adder(a)(a) for a in (0, -3, 10 ** 20)]", + def scaled(self, n: int) -> int: + return n * Consts.SCALE + + +class Empty: + pass + + +class Sub(Consts): + def twice(self, n: int) -> int: + return self.scaled(n) * 2 +", + &[ + "m.Helpers().doubled(21)", + "m.Helpers().summed(5)", + "m.Consts().scaled(4)", + "m.Consts.SCALE", + "m.Sub().twice(4)", + "isinstance(m.Sub(), m.Consts)", + "type(m.Empty()).__name__", + "type(_capture(lambda: m.Consts(1))).__name__", + "str(_capture(lambda: m.Consts(1))).rsplit('.', 1)[-1]", + "str(_capture(lambda: m.Empty(1, 2)))", + // and a class anything can be built on publishes no `__init__` the source + // did not write. a static type still does, because it has to fill the slot + // pair itself — but nothing can be built on one, so no mro reaches it + "'__init__' in vars(m.Consts)", + "[m.Helpers().doubled(n) for n in [0, 1, 2]]", ], ); } #[test] -fn a_loop_over_native_instances_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - // narrowing the element to an emitted class is a checked unbox like any other — - // asking the *free* narrowable rather than the one that knows this module's - // layouts declined the whole loop - agree( - "loopnative", +fn an_explicit_object_base_is_no_base_at_all() { + // `class C(object)` is what `class C:` already is, so it lays out and compiles the + // same way — but only when `object` really is the builtin + agree_python( + "objectbase", "\ -frozen data class Vec2: - x: float - y: float +class Plain(object): + def __init__(self, n: int) -> None: + self.n = n -data class Loose: - n: int + def doubled(self) -> int: + return self.n * 2 -def total(vs: list[Vec2]) -> float: - out = 0.0 - for v in vs: - out = out + v.x * v.x + v.y * v.y - return out -def summed(xs: list[Loose]) -> int: - return sum([x.n for x in xs]) +class Derived(Plain): + def __init__(self, n: int) -> None: + self.n = n + self.extra = 1 -def wrong_element(vs: list[Vec2]) -> float: - out = 0.0 - for v in vs: - out = out + v.x - return out + def total(self) -> int: + return self.doubled() + self.extra ", &[ - "round(m.total([m.Vec2(1.0, 2.0), m.Vec2(3.0, 4.0)]), 1)", - "m.summed([m.Loose(1), m.Loose(2)])", - "round(m.total([]), 1)", - // and the narrowing is a real check: a list that lies raises - "[type(e).__name__ for e in [_capture(m.wrong_element, ['not a vec'])]]", + "m.Plain(3).doubled()", + "m.Derived(4).total()", + "isinstance(m.Derived(1), m.Plain)", + "[m.Plain(n).doubled() for n in [0, 1, 2]]", + "m.Derived(2).n", ], ); } -/// the process's own peak footprint, which is the only thing that sees a buffer -/// that is not a `PyObject` -/// -/// windows has no `resource` module; `GetProcessMemoryInfo` answers the same -/// question there. the structure is spelled by field *width* rather than by C -/// name because `ctypes.c_ulong` is four bytes on windows and eight elsewhere — -/// a `DWORD` written as `c_ulong` would move every offset behind it — and its -/// size is checked on every platform, so the branch only windows takes is still -/// held to its layout here -const PEAK_FOOTPRINT: &str = "\ -import ctypes - -class _Counters(ctypes.Structure): - _fields_ = [(\"cb\", ctypes.c_uint32), (\"page_faults\", ctypes.c_uint32)] + [ - (name, ctypes.c_size_t) - for name in (\"peak_working_set\", \"working_set\", - \"quota_peak_paged_pool\", \"quota_paged_pool\", - \"quota_peak_non_paged_pool\", \"quota_non_paged_pool\", - \"pagefile\", \"peak_pagefile\")] +#[test] +fn a_module_that_binds_object_itself_keeps_its_own() { + // the name is resolved, not matched: a class of this module's own called `object` + // is the base, and taking the builtin instead would give the subclass the wrong one + agree_python( + "shadowedobject", + "\ +class object: + def __init__(self, n: int) -> None: + self.n = n -assert ctypes.sizeof(_Counters) == 8 + 8 * ctypes.sizeof(ctypes.c_size_t) + def base(self) -> int: + return self.n -def _windows_peak(): - kernel32 = ctypes.windll.kernel32 - kernel32.GetCurrentProcess.restype = ctypes.c_void_p - kernel32.K32GetProcessMemoryInfo.restype = ctypes.c_int32 - kernel32.K32GetProcessMemoryInfo.argtypes = ( - ctypes.c_void_p, ctypes.POINTER(_Counters), ctypes.c_uint32) - counters = _Counters() - counters.cb = ctypes.sizeof(counters) - if not kernel32.K32GetProcessMemoryInfo( - kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): - raise ctypes.WinError() - return counters.peak_working_set -try: - import resource -except ImportError: - _peak = _windows_peak -else: - def _peak(): - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss -"; +class Shadowed(object): + def __init__(self, n: int) -> None: + self.n = n + self.extra = 1 -#[test] -fn an_unboxed_array_does_not_leak_its_buffer() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_arrayleak"); - let _ = std::fs::remove_dir_all(&dir); - // the buffer is `PyMem_Malloc`, not a `PyObject` — so a leak of one is invisible - // to `gc.get_objects()` and to a refcount check. the process's own footprint is - // the only thing that sees it - let source = "\ -def churn(rounds: int) -> float: - out = 0.0 - r = 0 - while r < rounds: - xs = [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] - out = out + xs[0] - r = r + 1 - return out -"; - if build_source( - source, - "by_diff_arrayleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - let out = run( - &python, - &dir, - &format!( - "import by_diff_arrayleak as m\n{PEAK_FOOTPRINT}\ - m.churn(1000)\n\ - before = _peak()\n\ - m.churn(400000)\n\ - after = _peak()\n\ - print(after - before < before // 2 or after - before < 4_000_000)\n" - ), + def total(self) -> int: + return self.base() + self.extra +", + &[ + "m.Shadowed(3).total()", + "m.object(5).base()", + "isinstance(m.Shadowed(1), m.object)", + ], ); - assert_eq!(out, "True", "the array buffer leaks"); } #[test] -fn container_fast_paths_agree_and_respect_subclasses() { - // the fast paths are guarded on the *exact* type: a subclass may override - // `__getitem__` or `__missing__`, and a fast path that ignored that would be a - // wrong answer rather than a fast one - agree( - "containerprim", +fn a_parameter_defaulting_to_none_is_not_a_none_place() { + // `def f(x=None)` infers `Unknown | None`, and the gradual member is assignable to + // whatever is asked — so the union tested as assignable to `None` and the + // parameter got the `None` representation, which nothing else could be stored in. + // it is one of python's most common shapes + agree_python( + "nonedefault", "\ -def indexed(xs: list[int], i: int) -> int: - return xs[i] +def opened(f, mode=None): + if mode is None: + mode = 'rb' + return mode -def written(xs: list[int], i: int, v: int) -> str: - xs[i] = v - return str(xs) -def looked_up(d: dict[str, int], k: str) -> int: - return d[k] +def counted(items, start=None): + if start is None: + start = 0 + return start + len(items) -def sized(xs: object) -> int: - return len(xs) -def tupled(t: tuple[int, int], i: int) -> object: - return t[i] +def collected(seed=None): + if seed is None: + seed = [] + seed.append(1) + return seed ", &[ - "m.indexed([1, 2, 3], 1)", - "m.indexed([1, 2, 3], -1)", - "m.written([1, 2, 3], 0, 9)", - "m.looked_up({'a': 1}, 'a')", - "m.sized([1, 2, 3])", - "m.sized({'a': 1})", - "m.sized('abc')", - "m.tupled((4, 5), 1)", - // every error is python's own, message and class included - "[(type(e).__name__, str(e)) for e in [_capture(m.indexed, [1], 5)]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.written, [1], 5, 0)]]", - "[(type(e).__name__, repr(str(e))) for e in [_capture(m.looked_up, {}, 'zz')]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.tupled, (1,), 9)]]", - // a *subclass* overriding the protocol must still be honoured - "m.indexed(type('L', (list,), {'__getitem__': lambda s, i: 99})([1, 2]), 0)", - "m.looked_up(type('D', (dict,), {'__missing__': lambda s, k: 77})(), 'nope')", - "m.sized(type('L', (list,), {'__len__': lambda s: 42})([1]))", + "m.opened('x')", + "m.opened('x', 'wb')", + "m.counted([1, 2, 3])", + "m.counted([1, 2, 3], 10)", + "m.collected()", + "m.collected([9])", + "[m.opened('f', mode) for mode in [None, 'rb', 'wb']]", ], ); -} - -#[test] -fn a_comprehension_over_range_agrees() { - // the statement form has had a counting loop all along; the comprehension drove - // the iteration protocol instead — a `range` object, an iterator, a `next` and - // an unbox *per element*, for a loop whose bounds are right there - agree_with_declines( - "comprange", - "\ -def squares(n: int) -> object: - return [i * i for i in range(n)] +} -def filtered(n: int) -> object: - return [i for i in range(n) if i % 3 == 0] +#[test] +fn an_unannotated_parameter_with_a_default_is_not_a_float() { + // a parameter with no annotation infers `Unknown | `, and a + // *gradual* element is assignable both ways to everything — so one of them used to + // answer for both halves of the promotion test, and any `Unknown | T` read as + // `int | float`. that gave the parameter a `double` representation and a string + // default, which the c compiler rejected outright + agree_python( + "graduallead", + "\ +def failed(values, errmsg='negative value'): + for x in values: + if x < 0: + raise ValueError(errmsg) + yield x -def from_two(n: int) -> object: - return [i for i in range(2, n)] -def nested(n: int) -> object: - return [i * j for i in range(n) for j in range(n)] +def tagged(n, tag='t'): + return tag + str(n) -def buffered(n: int) -> float: - xs = [i * 1.5 for i in range(n)] - out = 0.0 - for x in xs: - out = out + x - return out -def empty(n: int) -> float: - xs = [i * 1.5 for i in range(0)] - out = 0.0 - for x in xs: - out = out + x - return out +def numeric(x, scale=2.0): + return x * scale ", &[ - "m.squares(5)", - "m.filtered(10)", - "m.from_two(6)", - "m.nested(3)", - "m.squares(0)", - "m.from_two(1)", - "m.buffered(5)", - "m.empty(0)", - // a negative bound is an empty range, not a backwards one - "m.squares(-3)", + "list(m.failed([1, 2], 'bad'))", + "list(m.failed([1, 2]))", + "[(type(e).__name__, str(e)) for e in [_capture(lambda: list(m.failed([1, -1])))]]", + "m.tagged(3)", + "m.tagged(3, 'x')", + "m.numeric(3)", + "m.numeric(3, 4.0)", + "m.numeric(2.5)", ], ); } #[test] -fn a_loop_over_an_unboxed_array_agrees() { - // the shape the whole representation exists for: an `i64` counter, no iterator - // object, no null test per step, and no bounds check — the counter is the - // lowering's own, so it is in range by construction - agree_with_declines( - "arrayloop", +fn a_gradual_value_narrowed_on_both_arms_is_still_an_object() { + // narrowing a gradual value gives an *intersection* holding it, so a conditional + // over both arms is a union of two of those and the gradual part sits one level + // down. it widens the whole type all the same: `(Unknown & C) | (Unknown & ~C)` is + // assignable to `None`, and reading that as a proof gave the result a `None` + // representation whose boundary rejected every value that was not one — which is + // how `enum.py` compiled and then could not be imported + agree_python( + "narrowedgradual", "\ -def total(n: int) -> float: - xs = [1.5, 2.5, 3.5, 4.5] - out = 0.0 - for x in xs: - out = out + x - return out - -def empty(n: int) -> float: - xs = [1.0] - out = 0.0 - for x in xs: - out = out + x - return out - -def broken(n: int) -> float: - xs = [1.0, 2.0, 3.0] - out = 0.0 - for x in xs: - if x > 1.5: - break - out = out + x - else: - out = -1.0 - return out - -def skipped(n: int) -> float: - xs = [1.0, 2.0, 3.0] - out = 0.0 - for x in xs: - if x > 1.5: - continue - out = out + x - return out +def unwrap(value): + return value.__func__ if isinstance(value, staticmethod) else value -def exhausted(n: int) -> float: - xs = [1.0, 2.0] - out = 0.0 - for x in xs: - out = out + x - else: - out = out + 100.0 - return out -def flags(n: int) -> int: - bs = [True, False, True, True] - total = 0 - for b in bs: - if b: - total = total + 1 - return total +def widest(value): + return value if isinstance(value, int) else value ", &[ - "m.total(0)", - "m.empty(0)", - // `break` skips the `else`, `continue` does not - "m.broken(0)", - "m.skipped(0)", - "m.exhausted(0)", - "m.flags(0)", + "m.unwrap(3)", + "m.unwrap('a')", + "m.unwrap(len).__name__", + "m.unwrap(staticmethod(len)).__name__", + "[m.widest(x) for x in (0, 'x', 2.5, None)]", + "m.widest(len).__name__", ], ); } #[test] -fn a_list_that_escapes_keeps_being_a_list() { - // the buffer is an optimization, not a restriction: a name that leaves the - // function never earns one in the first place, so it compiles exactly as it did - // before the representation existed - agree( - "bufferescape", +fn a_class_that_stores_through_setattr_keeps_its_interpreted_definition() { + // an emitted instance is its layout and nothing else — there is no `__dict__` + // behind it — so an attribute `setattr` names as a value has nowhere to go. the + // class has to stay interpreted, and the compiled module has to use *that* class: + // laying it out anyway is how `enum.py` reached `no __dict__ for setting new + // attributes` and could not be imported + agree_python_with_declines( + "setattrstore", "\ -def returned(n: int) -> object: - xs = [1.0, 2.0] - return xs +class Registry: + def __init__(self): + self.count = 0 -def passed(n: int) -> int: - xs = [1.0, 2.0] - return len(sorted(xs)) + def install(self, name, fn): + setattr(self, name, fn) + self.count = self.count + 1 -def stored(n: int) -> object: - xs = [1.0, 2.0] - return [xs, xs] + def run(self, name): + return getattr(self, name)() -def kept(n: int) -> float: - xs = [1.0, 2.0] - return xs[0] + xs[1] -def looped(n: int) -> float: - xs = [1.0, 2.0] - out = 0.0 - for x in xs: - out = out + x - return out +def build(): + r = Registry() + r.install('greet', lambda: 'hello') + return r ", &[ - // these keep a real list, and a real list is what comes back - "m.returned(0)", - "type(m.returned(0)).__name__", - "m.passed(0)", - "m.stored(0)", - // and these earn the buffer, invisibly - "m.kept(0)", - "m.looped(0)", + "m.build().run('greet')", + "m.build().count", + "[m.build().greet(), m.Registry().count]", ], ); } #[test] -fn a_from_import_inside_a_body_agrees() { - // plain python, so the interpreted leg is the source itself: the transpiler's - // lazy-import polyfill resolves `from pkg import submodule` wrongly, which would - // otherwise break the leg this is measured against rather than the compiled one +fn a_subscript_by_a_string_key_agrees() { + // a `str` widened to an `object` for a lookup must stay widened: substituting the + // source back handed `GetItem` a `str` register, which is a `PyObject *` in c and + // so compiled — but is a different representation, and the verifier said so agree_python( - "fromimport", + "strkey", "\ -def one(n: int) -> str: - from math import sqrt - return str(round(sqrt(n), 3)) +def read(d: dict[str, int], k: str) -> int: + return d[k] -def several(s: str) -> str: - from os.path import basename, dirname - return dirname(s) + '|' + basename(s) -def aliased(n: int) -> str: - from math import sqrt as root - return str(round(root(n), 3)) +def literal(d: dict[str, int]) -> int: + return d['k'] -# `urllib.parse` is not an attribute of `urllib` until something imports it, so -# this is the fromlist doing its job rather than an attribute read -def submodule(s: str) -> str: - from urllib import parse - return parse.quote(s) -def dotted(s: str) -> str: - from urllib.parse import quote - return quote(s) +def written(d: dict[str, int], k: str, v: int) -> object: + d[k] = v + return d -# a name the module does not have is an `ImportError`, not the `AttributeError` a -# plain attribute read would give — a guarded lazy import rests on the difference -def absent() -> object: - from os import definitely_not_a_real_name - return definitely_not_a_real_name -def guarded() -> str: - try: - from os import definitely_not_a_real_name - except ImportError: - return 'caught' - return 'no' +def missing(d: dict[str, int], k: str) -> object: + return d.get(k) ", &[ - "m.one(2)", - "m.several('/a/b/c.txt')", - "m.aliased(2)", - "m.submodule('a b/c')", - "m.dotted('a b/c')", - "m.guarded()", - // the type, the message and every attribute an `except` clause reads - "(lambda e: (type(e).__name__, str(e), e.name, e.path, e.name_from))(_capture(m.absent))", - // a thousand imports must not accumulate references to the module. a - // leaked module is one object with a climbing refcount, which no - // object-count check can see - "_repeated(lambda: m.one(2), 1000)", - "_refdelta('math', lambda: m.one(2), 500)", - "_refdelta('urllib.parse', lambda: m.submodule('a b'), 500)", + "m.read({'k': 1}, 'k')", + "m.literal({'k': 2})", + "m.written({}, 'a', 1)", + "m.missing({'k': 3}, 'k')", + "m.missing({'k': 3}, 'nope')", + "str(_capture(lambda: m.read({}, 'absent')))", + "[m.read({'a': 1, 'b': 2}, k) for k in ['a', 'b']]", + // the write goes straight to `PyDict_SetItem` for an *exact* dict, so a + // subclass that overrides `__setitem__` is what says the guard holds + "m.written(type('D', (dict,), \ + {'__setitem__': lambda s, k, v: dict.__setitem__(s, k, v * 10)})(), 'a', 5)", ], ); } #[test] -fn the_remaining_statement_and_expression_forms_agree() { +fn a_class_on_a_base_outside_the_module_agrees() { + // the type is built on whatever the name resolves to at module init, so the base + // has to *be* the real one: `raise` on a class that silently got `object` instead + // is a `TypeError`, which is how the two earlier attempts at this were caught + // + // the class declares no layout of its own — `basicsize` 0, and the base allocates + // and frees — and that has to hold **transitively**, or a subclass of a subclass + // declares a size smaller than its base and python rejects the type outright agree( - "coverage", + "external_base", "\ -def imports(n: int) -> str: - import math - return str(round(math.sqrt(n), 3)) +import collections -def aliased(n: int) -> str: - import os.path as p - return str(p.basename('/a/b')) +from collections import UserList -def del_key(d: dict[str, int]) -> str: - del d['a'] - return str(sorted(d.items())) -def del_item(xs: list[int]) -> str: - del xs[0] - return str(xs) +class MyError(Exception): + def label(self) -> str: + return \"mine\" -def del_attr(o: object) -> str: - del o.gone - return str(hasattr(o, 'gone')) -def ellipsis(n: int) -> object: - return ... +class Narrower(MyError): + def label(self) -> str: + return \"narrower\" -def negated(s: object) -> object: - return -s -def inverted(s: object) -> object: - return ~s +class Deeper(Narrower): + pass -def sliced(xs: list[int]) -> str: - return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[:2]) + str(xs[::-1]) -def slice_assigned(xs: list[int]) -> str: - xs[1:3] = [9, 9, 9] - return str(xs) +class Dotted(collections.OrderedDict): + # a base written as an attribute is a name to look up and then an attribute to + # walk, which is the only difference from a bare one + def label(self) -> str: + return \"dotted\" -def slice_deleted(xs: list[int]) -> str: - del xs[1:3] - return str(xs) -def walrus(xs: list[int]) -> str: - if (n := len(xs)) > 2: - return 'big ' + str(n) - return 'small ' + str(n) +class Listy(UserList): + def doubled(self) -> int: + return len(self) * 2 -def genexp(xs: list[int]) -> str: - return str(sum(x * 2 for x in xs)) + str(any(x > 2 for x in xs)) + str(max(x for x in xs)) -def declared_global(n: int) -> int: - global _counter - return n -", - &[ - "m.imports(16)", - "m.aliased(0)", - "m.del_key({'a': 1, 'b': 2})", - "m.del_item([1, 2, 3])", - "m.ellipsis(0)", - "m.sliced([1, 2, 3, 4])", - "m.slice_assigned([1, 2, 3, 4])", - "m.slice_deleted([1, 2, 3, 4])", - "m.walrus([1, 2, 3])", - "m.walrus([1])", - "m.genexp([1, 2, 3])", - "m.declared_global(4)", - // the protocol forms, on a type that answers them - "m.negated(type('N', (), {'__neg__': lambda s: 'neg'})())", - "m.inverted(type('N', (), {'__invert__': lambda s: 'inv'})())", - "m.del_attr(type('A', (), {})() if False else __import__('types').SimpleNamespace(gone=1))", - // and the errors each raises - "[(type(e).__name__, str(e)) for e in [_capture(m.del_key, {})]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.del_item, [])]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.negated, object())]]", - ], - ); -} +def raising(n: int) -> int: + if n < 0: + raise Narrower(\"negative\") + return n * 2 -#[test] -fn a_bad_first_argument_raises_rather_than_crashing() { - // the wrapper releases every argument local on the error path, so one whose - // declaration a `goto` skipped would be released while indeterminate. a wrong - // type in the *first* parameter is the reachable case: it jumps over the rest. - // - // only the exception *type* is compared: the boundary rejects a bad argument - // where the interpreted leg gets as far as the operation that uses it, so the - // two agree that it is a `TypeError` and not on where it was raised - agree( - "badarg", - "\ -def two(a: int, b: str) -> int: - return a + len(b) -def three(a: int, b: str, c: list[int]) -> int: - return a + len(b) + len(c) -", - &[ - "type(_capture(m.two, 'x', 'y')).__name__", - "type(_capture(m.two, 1, 2)).__name__", - "type(_capture(m.three, 'x', 'y', [1])).__name__", - "type(_capture(m.three, 1, 'y', 'z')).__name__", - "m.two(1, 'yy')", - "m.three(1, 'yy', [1, 2])", - ], - ); -} +def catching(n: int) -> str: + try: + return str(raising(n)) + except MyError as e: + return \"caught \" + str(e) + \" \" + e.label() -#[test] -fn a_plain_python_loop_closure_shares_its_binding() { - let Some((python, toolchain)) = environment() else { - return; - }; - // python's loop binding is shared by every iteration, so all three closures - // see the last value. basedpython's is per-iteration. the compiled half has to - // follow the *source* language, not a flag — a `.py` fallback is python - let source = "\ -def counters() -> list[object]: - out = [] - for i in range(3): - def get() -> int: - return i - out.append(get) - return [f() for f in out] -"; - let dir = std::env::temp_dir().join("by_diff_pyloop"); - let interpreted = std::env::temp_dir().join("by_diff_pyloop_i"); - let _ = std::fs::remove_dir_all(&dir); - let _ = std::fs::remove_dir_all(&interpreted); - std::fs::create_dir_all(&interpreted).expect("the directory is created"); - std::fs::write(interpreted.join("by_diff_pyloop.py"), source).expect("written"); - let options = Options { - language: by_irbuild::Language::Python, - ..Options::default() - }; - let built = match build_source(source, "by_diff_pyloop", &toolchain, &dir, &options) { - Ok(built) => built, - Err(error) => { - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - }; - assert!(built.declined.is_empty(), "{:?}", built.declined); +def hierarchy() -> str: + parts = ( + issubclass(Narrower, MyError), + issubclass(Deeper, Exception), + isinstance(Deeper(\"d\"), MyError), + Deeper(\"d\").label(), + ) + return \",\".join(str(x) for x in parts) - let body = "import by_diff_pyloop as m\nprint(m.counters())\n"; - let compiled = run(&python, &dir, body); - assert_eq!(compiled, run(&python, &interpreted, body)); - assert!(compiled.contains("[2, 2, 2]"), "{compiled}"); -} -#[test] -fn dunder_methods_fill_their_type_slots() { - // a method table cannot fill a slot: `repr(x)` reads `tp_repr` and never looks - // the name up. so each of these has to work *both* ways — through the slot and - // as an ordinary method - agree_python( - "dunders", - "\ -class Bag: - def __init__(self, items: list[int]) -> None: - self.items = items +def uses_list() -> str: + it = Listy([1, 2, 3]) + it.append(4) + return str(it.doubled()) + \" \" + str(len(it)) + \" \" + str(list(it)) - def __repr__(self) -> str: - return 'Bag(' + str(self.items) + ')' - def __str__(self) -> str: - return 'bag of ' + str(len(self.items)) +class WithStorage(Exception): + # a field of its own: the base allocates the instance, so this one lives in room + # asked for *past* it, and the class supplies the dealloc, traverse and clear that + # the base cannot write for storage it does not know about + def __init__(self, extra: int) -> None: + self.extra = extra - def __len__(self) -> int: - return len(self.items) + def bumped(self) -> int: + return self.extra + 1 - def __bool__(self) -> bool: - return len(self.items) > 1 + +class TwoBases(ValueError, IndexError): + # more than one base, none of them ours: python works out the mro and which of them + # owns the layout, and this class declares none of its own + def label(self) -> str: + return \"two\" + + +def caught_by_either(which: int) -> str: + try: + raise TwoBases(\"both\") + except IndexError as e: + if which == 0: + return \"index \" + str(e) + raise ", &[ - "repr(m.Bag([1, 2, 3]))", - "str(m.Bag([1, 2, 3]))", - "len(m.Bag([1, 2, 3]))", - "bool(m.Bag([1, 2, 3]))", - "bool(m.Bag([1]))", - "len(m.Bag([]))", - "bool(m.Bag([]))", - "f'{m.Bag([1])}'", - "'{}'.format(m.Bag([1, 2]))", - "m.Bag([1, 2]).__repr__()", - "m.Bag([1, 2]).__len__()", - "[b for b in [m.Bag([]), m.Bag([1, 2])] if b]", + "[m.catching(n) for n in (3, -1)]", + "m.hierarchy()", + "m.uses_list()", + "[type(e).__name__ for e in [_capture(m.raising, -2)]]", + "m.WithStorage(5).extra", + "m.WithStorage(5).bumped()", + "[type(e).__name__ for e in [_capture(m.WithStorage, 7)]]", + "(m.WithStorage(1).args, str(m.WithStorage(1)))", + "(m.Deeper.__mro__[1].__name__, m.Listy.__mro__[1].__name__)", + "m.caught_by_either(0)", + "[c.__name__ for c in m.TwoBases.__mro__[1:3]]", + "(issubclass(m.TwoBases, ValueError), issubclass(m.TwoBases, IndexError))", + "m.TwoBases('x').label()", + "(m.Dotted().label(), m.Dotted.__mro__[1].__name__)", ], ); } -#[test] -fn calling_a_resumable_frame_hands_back_the_state_object() { - // whatever the annotation says the body produces, the *call* gives the state - // object and the iteration or the `await` turns it back. there are three - // signature maps — module-level, methods, nested — and this has to hold in all of - // them, or the caller assigns a `PyObject *` into the annotation's representation - agree_python( - "resumablecall", - "\ -from typing import Any +/// the family a class appending storage grows: `TokenList` keeps a field past a `list` +/// instance, and every class under it keeps that one and adds none +/// +/// this is the stdlib's own shape — `email._header_value_parser` writes thirty-seven of +/// them — and the whole point is that a subclass adding no field of its own appends +/// nothing: what it stores is what `TokenList` stores, at the offset `TokenList` laid it +/// out and through the descriptor `TokenList` published. `Restating` is the same class +/// written the other way round, assigning an attribute the base already keeps +const APPENDED_FAMILY: &str = "\ +class TokenList(list): + + token_type = None + def __init__(self, *args): + super().__init__(*args) + self.defects = [] -def flags(n): - i = 0 - while i < n: - yield i > 0 - i = i + 1 + def kind(self): + return self.token_type + def defect_count(self): + return len(self.defects) -async def doubled(n: int) -> int: - return n * 2 +class Plain(TokenList): + pass -class Holder: - def __init__(self, n: int) -> None: - self.n = n - def counted(self): - i = 0 - while i < self.n: - yield i == 0 - i = i + 1 +class Named(TokenList): + token_type = 'named' - def total(self) -> int: - out = 0 - for seen in self.counted(): - if seen: - out = out + 1 - return out +class Deeper(Named): + token_type = 'deeper' -def nested(n: int) -> int: - def inner(): - i = 0 - while i < n: - yield i < 1 - i = i + 1 - out = 0 - for seen in inner(): - if seen: - out = out + 1 - return out + def kind(self): + return 'deep:' + str(self.token_type) -def drained(n: int) -> int: - out = 0 - for seen in flags(n): - if seen: - out = out + 1 - return out -", +class Restating(TokenList): + def __init__(self, *args): + super().__init__(*args) + self.defects = ['restated'] +"; + +#[test] +fn a_subclass_that_appends_nothing_past_a_base_agrees() { + agree_python( + "appendnothing", + APPENDED_FAMILY, &[ - "list(m.flags(3))", - "m.drained(3)", - "m.drained(0)", - "_run(m.doubled(5))", - "m.Holder(4).total()", - "list(m.Holder(2).counted())", - "m.nested(3)", - "m.nested(0)", + "[(type(t).__name__, list(t), t.defects, t.kind(), t.defect_count())\n\ + \x20 for t in (m.TokenList([0]), m.Plain([1]), m.Named([2]), m.Deeper([3]),\n\ + \x20 m.Restating([4]))]", + // the base's field, written and read through a subclass instance + "[(p.defects.append('one'), p.defects, p.defect_count(),\n\ + \x20 (p.append(9), list(p))[1]) for p in [m.Plain([1, 2])]]", + "[c.__name__ for c in m.Deeper.__mro__]", + "(isinstance(m.Deeper([1]), m.TokenList), isinstance(m.Restating([1]), list),\n\ + \x20issubclass(m.Plain, m.TokenList))", + "(m.Plain.token_type, m.Named.token_type, m.Deeper.token_type)", + "(sorted(m.Plain([3, 1, 2])), m.Plain([1, 2]) == [1, 2], m.Plain([1]) + [2])", + // python's own subclass of one, built by the class statement rather than here + "[(list(s([5])), s([5]).defects, s([5]).kind())\n\ + \x20 for s in [type('Py', (m.Plain,), {'token_type': 'py'})]]", + // every instance holds a cycle through the appended field, so the collector + // has to be able to see it and the deallocation has to release it + "(len([t for t in [m.Deeper([i]) for i in range(200)]\n\ + \x20 if t.defects.append(t) is None]), __import__('gc').collect() >= 0)", ], ); } #[test] -fn a_class_of_only_methods_compiles() { - // no `__init__` is not the same as no *layout*: a class of methods has an empty - // one, which is as representable as any other. `object.__init__` is what rejects a - // call with arguments, and it names the class rather than a method it does not have +fn a_subclass_that_appends_nothing_is_the_compiled_type() { + // the behaviour above is answered identically by a class that fell back to its + // interpreted definition, so it cannot say which build answered. + // `method_descriptor` can: a real type of ours holds one where the interpreted + // class holds a plain function + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_appendnothing_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + APPENDED_FAMILY, + "by_diff_appendnothing_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_appendnothing_t as m\n\ + print(type(m.TokenList.__dict__['kind']).__name__,\n\ + \x20 type(m.Deeper.__dict__['kind']).__name__)\n\ + print(m.TokenList.__basicsize__ == m.Plain.__basicsize__\n\ + \x20 == m.Named.__basicsize__ == m.Deeper.__basicsize__)\n\ + print(m.Deeper([1]).defects, m.Restating([1]).defects)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + True\n\ + [] ['restated']" + ); +} + +#[test] +fn a_subclass_with_no_storage_stands_on_a_base_that_declined_later() { + // a base is settled as one of ours while the layouts settle, and only the body being + // lowered can turn it down after that — here a `__new__`, which fills a type slot + // with no adapter. a class with no storage of its own does not need the base to have + // stayed one: what stands under the name at import is a class either way, so it is + // built on the *name*, which is the construction every class over an outside base + // already takes. // - // `Empty` is where the class is a *static* type, which nothing can build on, so it - // fills the pair of slots itself and raises through them. `Consts` is a base, so it - // is a heap type and leaves both to `object` — where the message carries the module, - // because a type built from a spec keeps its module in `tp_name`. that difference is - // the price of not publishing an `__init__` the source never wrote: one installed - // anyway stands between everything built on the class and the `object.__init__` it - // should have reached - agree_python( - "onlymethods", - "\ -class Helpers: - def doubled(self, n: int) -> int: - return n * 2 + // `method_descriptor` against `function` is what says which type answered: `Plain` + // and `Deeper` are compiled types standing on the interpreted `TokenList` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_lostbase"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class TokenList(list): - def summed(self, n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + i - i = i + 1 - return total + token_type = None + def __new__(cls, *args): + return super().__new__(cls, *args) -class Consts: - SCALE = 3 + def __init__(self, *args): + super().__init__(*args) + self.defects = [] - def scaled(self, n: int) -> int: - return n * Consts.SCALE + def kind(self): + return self.token_type -class Empty: - pass +class Plain(TokenList): + token_type = 'plain' + def side(self): + return 'side:' + str(self.token_type) -class Sub(Consts): - def twice(self, n: int) -> int: - return self.scaled(n) * 2 -", - &[ - "m.Helpers().doubled(21)", - "m.Helpers().summed(5)", - "m.Consts().scaled(4)", - "m.Consts.SCALE", - "m.Sub().twice(4)", - "isinstance(m.Sub(), m.Consts)", - "type(m.Empty()).__name__", - "type(_capture(lambda: m.Consts(1))).__name__", - "str(_capture(lambda: m.Consts(1))).rsplit('.', 1)[-1]", - "str(_capture(lambda: m.Empty(1, 2)))", - // and a class anything can be built on publishes no `__init__` the source - // did not write. a static type still does, because it has to fill the slot - // pair itself — but nothing can be built on one, so no mro reaches it - "'__init__' in vars(m.Consts)", - "[m.Helpers().doubled(n) for n in [0, 1, 2]]", - ], + +class Deeper(Plain): + token_type = 'deeper' +"; + let built = match build_source( + source, + "by_diff_lostbase", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![( + "TokenList", + "`__new__` fills a type slot with no adapter yet" + )] + ); + let out = run( + &python, + &dir, + "import by_diff_lostbase as m\n\ + print([c.__name__ for c in m.Deeper.__mro__])\n\ + print(type(m.Plain.__dict__['side']).__name__,\n\ + \x20 type(m.TokenList.__dict__['kind']).__name__)\n\ + print(list(m.Plain([1, 2])), m.Plain([1, 2]).defects, m.Plain([1]).side())\n\ + print(m.Deeper([3]).kind(), m.Deeper([3]).side(), isinstance(m.Deeper([3]), m.TokenList))\n", + ); + assert_eq!( + out, + "['Deeper', 'Plain', 'TokenList', 'list', 'object']\n\ + method_descriptor function\n\ + [1, 2] [] side:plain\n\ + deeper side:deeper True" ); } #[test] -fn an_explicit_object_base_is_no_base_at_all() { - // `class C(object)` is what `class C:` already is, so it lays out and compiles the - // same way — but only when `object` really is the builtin +fn a_base_this_module_emits_beside_one_it_does_not_agrees() { + // a class may hold both kinds of base at once. it takes its whole layout from + // outside — the base of ours in the list lays nothing out, so it asks for no room — + // and python works out the mro and which of the bases owns the instance. + // + // the outside base may own a real one: `dict`, `int` and `Exception` each decide the + // instance, and getting that wrong writes this class's idea of a layout over theirs agree_python( - "objectbase", + "mixed_bases", "\ -class Plain(object): - def __init__(self, n: int) -> None: - self.n = n +import codecs - def doubled(self) -> int: - return self.n * 2 +class Ours: + def side(self) -> str: + return \"ours\" -class Derived(Plain): - def __init__(self, n: int) -> None: - self.n = n - self.extra = 1 + def reset(self) -> object: + # `object`, because the outside base's `reset` answers `None` and a narrower + # annotation would make the difference a representation error rather than a value + return \"ours reset\" + + +class OursFirst(Ours, codecs.Codec): + def label(self) -> str: + return \"first\" + + +class OutsideFirst(codecs.StreamWriter, Ours): + # the outside base comes first, so *its* `reset` is what the mro reaches and ours is + # shadowed — a direct call on a receiver typed as `Ours` would answer with ours + def label(self) -> str: + return \"outside\" + + +class AsDict(dict, Ours): + def label(self) -> str: + return \"dict\" + + +class AsInt(int, Ours): + def label(self) -> str: + return \"int\" + + +class OurError(Exception): + def which(self) -> str: + return \"ourerror\" + + +class Diamond(OurError, ValueError): + # the shared ancestor is outside: `Exception` is above both legs, and the mro has to + # place it once and after both + def label(self) -> str: + return \"diamond\" + + +class Under(OursFirst): + # a class of this module's own built on the mixture: the layout chain runs through + # a class that has none, so this one declares none either + def label(self) -> str: + return \"under\" - def total(self) -> int: - return self.doubled() + self.extra -", - &[ - "m.Plain(3).doubled()", - "m.Derived(4).total()", - "isinstance(m.Derived(1), m.Plain)", - "[m.Plain(n).doubled() for n in [0, 1, 2]]", - "m.Derived(2).n", - ], - ); -} -#[test] -fn a_module_that_binds_object_itself_keeps_its_own() { - // the name is resolved, not matched: a class of this module's own called `object` - // is the base, and taking the builtin instead would give the subclass the wrong one - agree_python( - "shadowedobject", - "\ -class object: - def __init__(self, n: int) -> None: - self.n = n +def through_the_base(o: Ours) -> str: + return o.side() - def base(self) -> int: - return self.n +def resetting(o: Ours) -> object: + return o.reset() -class Shadowed(object): - def __init__(self, n: int) -> None: - self.n = n - self.extra = 1 - def total(self) -> int: - return self.base() + self.extra +def exactly(which: int) -> str: + if which == 0: + return OursFirst().side() + return OutsideFirst(None).side() ", &[ - "m.Shadowed(3).total()", - "m.object(5).base()", - "isinstance(m.Shadowed(1), m.object)", + "[c.__name__ for c in m.OursFirst.__mro__]", + "[c.__name__ for c in m.OutsideFirst.__mro__]", + "[c.__name__ for c in m.Diamond.__mro__]", + "(m.OursFirst.__base__.__name__, m.AsDict.__base__.__name__, m.AsInt.__base__.__name__)", + "(m.OursFirst().side(), m.OursFirst().label())", + "(m.OutsideFirst(None).side(), m.OutsideFirst(None).label())", + "[m.through_the_base(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None), m.Under())]", + "[m.exactly(n) for n in (0, 1)]", + "[m.resetting(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None))]", + "([c.__name__ for c in m.Under.__mro__], m.Under().label(), m.Under().side())", + // the outside base still owns the instance it always owned + "(sorted(m.AsDict(a=1, b=2).items()), m.AsDict().label())", + "(int(m.AsInt(7)) + 1, m.AsInt(7).label(), m.AsInt(7).side())", + "(m.Diamond('boom').args, str(m.Diamond('boom')), m.Diamond('x').which(), m.Diamond('x').label())", + "(isinstance(m.Diamond('x'), ValueError), isinstance(m.Diamond('x'), m.OurError))", + // an instance of the mixture carries whatever `__dict__` the outside base + // brought: a class of ours alone has none, and losing it here silently + // turned every attribute read on one into a walk off the end of the object + "(lambda o: (setattr(o, 'kept', 3), o.kept, o.__dict__))(m.OursFirst())", + // and a python subclass of the result, which is a class statement on it + "(lambda C: (C().side(), C().label(), [c.__name__ for c in C.__mro__]))\ + (type('Py', (m.OursFirst,), {'label': lambda self: 'py'}))", ], ); } #[test] -fn a_parameter_defaulting_to_none_is_not_a_none_place() { - // `def f(x=None)` infers `Unknown | None`, and the gradual member is assignable to - // whatever is asked — so the union tested as assignable to `None` and the - // parameter got the `None` representation, which nothing else could be stored in. - // it is one of python's most common shapes - agree_python( - "nonedefault", - "\ -def opened(f, mode=None): - if mode is None: - mode = 'rb' - return mode +fn a_base_beside_an_outside_one_is_built_by_calling_its_metaclass() { + // a type spec takes its whole instance shape from the one base python picks out of + // the list. where that is a class of ours the `__dict__` an outside base needs is + // dropped, and the type then claims a managed dict it has no room for — the first + // attribute read on an instance walks off the object and segfaults. calling the + // metaclass works the shape out from every base at once + // + // the descriptors say which build answered: a class that fell back to its + // interpreted definition would agree on every value above and say `function` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_mixedmeta"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import abc +import codecs -def counted(items, start=None): - if start is None: - start = 0 - return start + len(items) +class Ours: + def side(self) -> str: + return \"ours\" -def collected(seed=None): - if seed is None: - seed = [] - seed.append(1) - return seed -", - &[ - "m.opened('x')", - "m.opened('x', 'wb')", - "m.counted([1, 2, 3])", - "m.counted([1, 2, 3], 10)", - "m.collected()", - "m.collected([9])", - "[m.opened('f', mode) for mode in [None, 'rb', 'wb']]", - ], +class Mixed(Ours, codecs.Codec): + def label(self) -> str: + return \"mixed\" + + +class Metaclassed(Ours, abc.ABC): + # a base whose metaclass is not `type` rules the spec out on its own, and the + # keyword the header carries has nowhere to go in one either + def label(self) -> str: + return \"meta\" +"; + let built = match build_source( + source, + "by_diff_mixedmeta", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_mixedmeta as m\n\ + print(type(m.Mixed.label).__name__, type(m.Ours.side).__name__)\n\ + print(m.Mixed.__base__.__name__, type(m.Metaclassed).__name__)\n\ + o = m.Mixed()\n\ + o.kept = 4\n\ + print(o.side(), o.label(), o.kept, o.__dict__)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + Ours ABCMeta\n\ + ours mixed 4 {'kept': 4}" ); } #[test] -fn an_unannotated_parameter_with_a_default_is_not_a_float() { - // a parameter with no annotation infers `Unknown | `, and a - // *gradual* element is assignable both ways to everything — so one of them used to - // answer for both halves of the promotion test, and any `Unknown | T` read as - // `int | float`. that gave the parameter a `double` representation and a string - // default, which the c compiler rejected outright - agree_python( - "graduallead", +fn a_class_built_through_its_metaclass_agrees() { + // `PyType_FromSpecWithBases` gives the type it builds `type` for a metaclass, so it + // cannot be handed a base with any other one, and `PyType_FromMetaclass` refuses a + // metaclass that overrides `__new__` — which `ABCMeta` does. so the type is built + // the way python builds one: by calling the metaclass with a namespace. + // + // the methods go *in* that namespace rather than onto the finished type, which is + // what fills the type slots — `type.__new__` runs the same fixup a class statement + // does, so `__repr__` becomes `tp_repr` with no adapter of ours + agree( + "metaclass", "\ -def failed(values, errmsg='negative value'): - for x in values: - if x < 0: - raise ValueError(errmsg) - yield x +from abc import ABC, ABCMeta -def tagged(n, tag='t'): - return tag + str(n) +class Node(ABC): + def kind(self) -> str: + return \"node\" + def __repr__(self) -> str: + return \"Node(\" + self.kind() + \")\" -def numeric(x, scale=2.0): - return x * scale -", - &[ - "list(m.failed([1, 2], 'bad'))", - "list(m.failed([1, 2]))", - "[(type(e).__name__, str(e)) for e in [_capture(lambda: list(m.failed([1, -1])))]]", - "m.tagged(3)", - "m.tagged(3, 'x')", - "m.numeric(3)", - "m.numeric(3, 4.0)", - "m.numeric(2.5)", - ], - ); -} + def __len__(self) -> int: + return 3 -#[test] -fn a_gradual_value_narrowed_on_both_arms_is_still_an_object() { - // narrowing a gradual value gives an *intersection* holding it, so a conditional - // over both arms is a union of two of those and the gradual part sits one level - // down. it widens the whole type all the same: `(Unknown & C) | (Unknown & ~C)` is - // assignable to `None`, and reading that as a proof gave the result a `None` - // representation whose boundary rejected every value that was not one — which is - // how `enum.py` compiled and then could not be imported - agree_python( - "narrowedgradual", - "\ -def unwrap(value): - return value.__func__ if isinstance(value, staticmethod) else value + +class Deeper(Node): + # a base of *ours* whose own base is not: the layout is nobody's, and the metaclass + # is inherited down the chain + def kind(self) -> str: + return \"deeper\" -def widest(value): - return value if isinstance(value, int) else value -", - &[ - "m.unwrap(3)", - "m.unwrap('a')", - "m.unwrap(len).__name__", - "m.unwrap(staticmethod(len)).__name__", - "[m.widest(x) for x in (0, 'x', 2.5, None)]", - "m.widest(len).__name__", - ], - ); -} +class Keyed(metaclass=ABCMeta): + # no bases at all, only the keyword — python supplies `(object,)` itself + def kind(self) -> str: + return \"keyed\" -#[test] -fn a_class_that_stores_through_setattr_keeps_its_interpreted_definition() { - // an emitted instance is its layout and nothing else — there is no `__dict__` - // behind it — so an attribute `setattr` names as a value has nowhere to go. the - // class has to stay interpreted, and the compiled module has to use *that* class: - // laying it out anyway is how `enum.py` reached `no __dict__ for setting new - // attributes` and could not be imported - agree_python_with_declines( - "setattrstore", - "\ -class Registry: - def __init__(self): - self.count = 0 - def install(self, name, fn): - setattr(self, name, fn) - self.count = self.count + 1 +class Checked(ABC): + # an `__init__` of its own that stores nothing: it has to reach `tp_init`, which on + # this construction is python's dispatcher reading the namespace entry + def __init__(self, value: int) -> None: + if value < 0: + raise ValueError(\"negative\") - def run(self, name): - return getattr(self, name)() + +class Equated(ABC): + # a class defining `__eq__` and not `__hash__` is unhashable. a spec has to be told + # that; `type.__new__` does it for itself, so this is the same answer reached two + # different ways depending on which construction the bases allowed + def __eq__(self, other: object) -> bool: + return isinstance(other, Equated) -def build(): - r = Registry() - r.install('greet', lambda: 'hello') - return r +def widened(value: object) -> str: + if isinstance(value, Node): + return \"node \" + value.kind() + return \"other\" ", &[ - "m.build().run('greet')", - "m.build().count", - "[m.build().greet(), m.Registry().count]", + // the metaclass is the one python would have worked out, not `type` + "(type(m.Node).__name__, type(m.Keyed).__name__, type(m.Deeper).__name__)", + // and the whole ancestry with it + "[c.__name__ for c in m.Node.__mro__]", + "[c.__name__ for c in m.Keyed.__mro__]", + "[c.__name__ for c in m.Deeper.__mro__]", + // an abstract base answers for it, which is the point of the metaclass + "(isinstance(m.Node(), __import__('abc').ABC), issubclass(m.Node, __import__('abc').ABC))", + "(isinstance(m.Deeper(), m.Node), issubclass(m.Deeper, m.Node))", + // `ABCMeta.__new__` ran, and ran over *this* class's namespace + "sorted(m.Node.__abstractmethods__)", + // a dunder written in the class body fills its slot + "repr(m.Node())", + "repr(m.Deeper())", + "len(m.Node())", + // and the ordinary methods still bind + "(m.Node().kind(), m.Deeper().kind(), m.Keyed().kind())", + "[m.widened(v) for v in (m.Node(), m.Deeper(), 1)]", + // a written `__init__` runs, and the errors around it are python's own + "(m.Checked(1) is not None, type(_capture(m.Checked, -1)).__name__)", + "type(_capture(m.Checked)).__name__", + // and `__eq__` without `__hash__` still takes the hash away + "(m.Equated() == m.Equated(), m.Equated() == 1)", + "type(_capture(hash, m.Equated())).__name__", + // the class reports the module it was written in, not `builtins` + "(m.Node.__module__, m.Keyed.__module__)", + "(m.Node.__name__, m.Node.__qualname__)", + // an imported base reaches this module as a lazy proxy, so `__mro_entries__` + // is what turns it into the class — and python records what was written + "repr(m.Node.__orig_bases__)", ], ); } -#[test] -fn a_subscript_by_a_string_key_agrees() { - // a `str` widened to an `object` for a lookup must stay widened: substituting the - // source back handed `GetItem` a `str` register, which is a `PyObject *` in c and - // so compiled — but is a different representation, and the verifier said so - agree_python( - "strkey", - "\ -def read(d: dict[str, int], k: str) -> int: - return d[k] - - -def literal(d: dict[str, int]) -> int: - return d['k'] +/// a class inside a package reports the whole dotted module it was written in +/// +/// the test above asks this of a top-level module, where the module's own name and +/// its name inside its package are the same string — so it cannot see the +/// difference. cpython reads a type's `__module__` off the front of its `tp_name` +/// and its `__name__` off the back, and a compiled class that carried only its +/// file's stem named a module `sys.modules` has nothing under. `dataclasses` looks +/// exactly that up (`sys.modules.get(cls.__module__).__dict__`), so a package +/// member with a dataclass in it failed to import at all +#[test] +fn a_class_in_a_package_reports_the_package_it_came_from() { + let Some((python, toolchain)) = environment() else { + return; + }; + let source = "\ +class Point: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + def total(self) -> int: + return self.x + self.y +"; + let base = std::env::temp_dir().join("by_diff_pkgmodule"); + let _ = std::fs::remove_dir_all(&base); + let compiled_root = base.join("c"); + let interpreted_root = base.join("i"); + let compiled = compiled_root.join("by_diff_pkg"); + let interpreted = interpreted_root.join("by_diff_pkg"); + for dir in [&compiled, &interpreted] { + std::fs::create_dir_all(dir).expect("the package directory is created"); + std::fs::write(dir.join("__init__.py"), "").expect("the package marker is written"); + } + std::fs::write(interpreted.join("member.py"), source) + .expect("the interpreted module is written"); -def written(d: dict[str, int], k: str, v: int) -> object: - d[k] = v - return d + // the *root* of the output tree, not the package directory: the build writes the + // artefact at the module's own place within the tree, so handing it the package + // directory would nest a second `by_diff_pkg` inside the first + let built = match build_source( + source, + "by_diff_pkg.member", + &toolchain, + &compiled_root, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let body = "\ +import sys\n\ +import by_diff_pkg.member as m\n\ +print(m.Point.__module__, m.Point.__qualname__, m.Point.__name__)\n\ +print(sys.modules[m.Point.__module__] is m)\n\ +print(m.Point(2, 3).total())\n"; + let compiled_out = run(&python, &compiled_root, body); + let interpreted_out = run(&python, &interpreted_root, body); + assert_eq!( + compiled_out, interpreted_out, + "compiled {compiled_out}, interpreted {interpreted_out}" + ); + assert_eq!( + compiled_out, "by_diff_pkg.member Point Point\nTrue\n5", + "the module python imported is the module the class names" + ); -def missing(d: dict[str, int], k: str) -> object: - return d.get(k) -", - &[ - "m.read({'k': 1}, 'k')", - "m.literal({'k': 2})", - "m.written({}, 'a', 1)", - "m.missing({'k': 3}, 'k')", - "m.missing({'k': 3}, 'nope')", - "str(_capture(lambda: m.read({}, 'absent')))", - "[m.read({'a': 1, 'b': 2}, k) for k in ['a', 'b']]", - // the write goes straight to `PyDict_SetItem` for an *exact* dict, so a - // subclass that overrides `__setitem__` is what says the guard holds - "m.written(type('D', (dict,), \ - {'__setitem__': lambda s, k, v: dict.__setitem__(s, k, v * 10)})(), 'a', 5)", - ], + // …and it really was the emitted type that answered. a class that fell back to + // its interpreted definition answers all of the above identically, and holds a + // plain function where a type of ours holds a descriptor + let descriptor = run( + &python, + &compiled_root, + "import by_diff_pkg.member as m\n\ + print(type(m.Point.__dict__['total']).__name__)\n", ); + assert_eq!(descriptor, "method_descriptor"); } #[test] -fn a_class_on_a_base_outside_the_module_agrees() { - // the type is built on whatever the name resolves to at module init, so the base - // has to *be* the real one: `raise` on a class that silently got `object` instead - // is a `TypeError`, which is how the two earlier attempts at this were caught - // - // the class declares no layout of its own — `basicsize` 0, and the base allocates - // and frees — and that has to hold **transitively**, or a subclass of a subclass - // declares a size smaller than its base and python rejects the type outright +fn a_class_keyword_reaches_init_subclass_agrees() { + // a keyword is the metaclass's business, and a type spec has nowhere to put one. + // the base is built at runtime because a class this module *emits* would have its + // layout laid out here, which the metaclass construction gives up agree( - "external_base", + "classkeyword", "\ -import collections - -from collections import UserList - - -class MyError(Exception): - def label(self) -> str: - return \"mine\" - - -class Narrower(MyError): - def label(self) -> str: - return \"narrower\" - - -class Deeper(Narrower): - pass - +from abc import ABCMeta -class Dotted(collections.OrderedDict): - # a base written as an attribute is a name to look up and then an attribute to - # walk, which is the only difference from a bare one - def label(self) -> str: - return \"dotted\" +def _tagged(cls: type, tag: str = \"none\", ready: bool = False, level: int = 0) -> None: + cls.tag = tag + cls.ready = ready + cls.level = level -class Listy(UserList): - def doubled(self) -> int: - return len(self) * 2 +Base = ABCMeta(\"Base\", (), {\"__init_subclass__\": classmethod(_tagged)}) -def raising(n: int) -> int: - if n < 0: - raise Narrower(\"negative\") - return n * 2 +class Alpha(Base, tag=\"alpha\"): + def kind(self) -> str: + return \"alpha\" -def catching(n: int) -> str: - try: - return str(raising(n)) - except MyError as e: - return \"caught \" + str(e) + \" \" + e.label() +class Beta(Base, tag=\"beta\", ready=True, level=2): + # a literal keyword as well as a name: both are evaluated where a class body would + # have evaluated them + def kind(self) -> str: + return \"beta\" -def hierarchy() -> str: - parts = ( - issubclass(Narrower, MyError), - issubclass(Deeper, Exception), - isinstance(Deeper(\"d\"), MyError), - Deeper(\"d\").label(), - ) - return \",\".join(str(x) for x in parts) +# the keyword's own text is arbitrary, and the C string form of it stopped at a NUL +class Gamma(Base, tag=\"a\\x00b\"): + def kind(self) -> str: + return \"gamma\" +", + &[ + // the keyword reached `__init_subclass__`, which the metaclass's `__new__` + // is what calls + "(m.Alpha.tag, m.Alpha.ready, m.Alpha.level)", + "(m.Beta.tag, m.Beta.ready, m.Beta.level)", + "ascii(m.Gamma.tag)", + "(m.Alpha().kind(), m.Beta().kind(), m.Gamma().kind())", + // and the metaclass itself came from the base + "(type(m.Alpha).__name__, type(m.Beta).__name__)", + "[c.__name__ for c in m.Beta.__mro__]", + "sorted(m.Beta.__abstractmethods__)", + ], + ); +} -def uses_list() -> str: - it = Listy([1, 2, 3]) - it.append(4) - return str(it.doubled()) + \" \" + str(len(it)) + \" \" + str(list(it)) +#[test] +fn which_build_answers_for_a_metaclass_class() { + // the two `agree` tests above cannot see this on their own: a class that quietly + // fell back to its interpreted definition answers *identically*, so both would pass + // on a compiler that built nothing at all. this one names the build — a compiled + // method is a descriptor, an interpreted one is a plain function. + // + // it is also where the restriction is pinned down. the construction through a + // metaclass hands back a type whose instance layout is the metaclass's answer, so a + // class appending fields to its base has nowhere to put them and stays interpreted; + // its fieldless sibling, on the same base, does not + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metafields"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABC, ABCMeta -class WithStorage(Exception): - # a field of its own: the base allocates the instance, so this one lives in room - # asked for *past* it, and the class supplies the dealloc, traverse and clear that - # the base cannot write for storage it does not know about - def __init__(self, extra: int) -> None: - self.extra = extra +class Fielded(ABC): + def __init__(self, value: int) -> None: + self.value = value - def bumped(self) -> int: - return self.extra + 1 + def doubled(self) -> int: + return self.value * 2 -class TwoBases(ValueError, IndexError): - # more than one base, none of them ours: python works out the mro and which of them - # owns the layout, and this class declares none of its own +class Fieldless(ABC): def label(self) -> str: - return \"two\" + return \"fieldless\" -def caught_by_either(which: int) -> str: - try: - raise TwoBases(\"both\") - except IndexError as e: - if which == 0: - return \"index \" + str(e) - raise -", - &[ - "[m.catching(n) for n in (3, -1)]", - "m.hierarchy()", - "m.uses_list()", - "[type(e).__name__ for e in [_capture(m.raising, -2)]]", - "m.WithStorage(5).extra", - "m.WithStorage(5).bumped()", - "[type(e).__name__ for e in [_capture(m.WithStorage, 7)]]", - "(m.WithStorage(1).args, str(m.WithStorage(1)))", - "(m.Deeper.__mro__[1].__name__, m.Listy.__mro__[1].__name__)", - "m.caught_by_either(0)", - "[c.__name__ for c in m.TwoBases.__mro__[1:3]]", - "(issubclass(m.TwoBases, ValueError), issubclass(m.TwoBases, IndexError))", - "m.TwoBases('x').label()", - "(m.Dotted().label(), m.Dotted.__mro__[1].__name__)", - ], +class Keyed(metaclass=ABCMeta): + def label(self) -> str: + return \"keyed\" +"; + let built = match build_source( + source, + "by_diff_metafields", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![( + "Fielded", + "a class with fields of its own needs a base whose metaclass is `type`" + )] + ); + let out = run( + &python, + &dir, + "import by_diff_metafields as m\n\ + print(m.Fielded(4).doubled(), m.Fielded(4).value)\n\ + print(m.Fieldless().label(), m.Keyed().label())\n\ + print(type(m.Fielded).__name__, type(m.Fieldless).__name__, type(m.Keyed).__name__)\n\ + print(isinstance(m.Fielded(1), m.Fielded), isinstance(m.Fieldless(), m.Fieldless))\n\ + print(type(m.Fielded.doubled).__name__, type(m.Fieldless.label).__name__,\n\ + \x20 type(m.Keyed.label).__name__)\n", + ); + assert_eq!( + out, + "8 4\n\ + fieldless keyed\n\ + ABCMeta ABCMeta ABCMeta\n\ + True True\n\ + function method_descriptor method_descriptor" ); } #[test] -fn a_base_this_module_emits_beside_one_it_does_not_agrees() { - // a class may hold both kinds of base at once. it takes its whole layout from - // outside — the base of ours in the list lays nothing out, so it asks for no room — - // and python works out the mro and which of the bases owns the instance. +fn a_metaclass_that_remakes_a_class_level_constant_is_turned_down_after_the_call() { + // the constants go into the namespace the metaclass is handed, which is enough for a + // metaclass that only *reads* one. an `EnumType` does not read them: it builds a + // *member* out of `STRICT = auto()`, and the member is not the value the module body + // already took a reference to. `FIRST is Boundary.STRICT` is what that costs, and no + // amount of writing the namespace fixes it. // - // the outside base may own a real one: `dict`, `int` and `Exception` each decide the - // instance, and getting that wrong writes this class's idea of a layout over theirs - agree_python( - "mixed_bases", - "\ -import codecs + // so the class is asked afterwards whether it kept what it was handed, and where it + // did not the interpreted definition stands. the two classes are the boundary: same + // base, same metaclass, and only `Boundary` has a constant. `function` against + // `method_descriptor` is what says the refusal is exactly that narrow — a guard that + // never fired would make `Boundary` say `method_descriptor` and lose `FIRST` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaconstant"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from enum import StrEnum, auto -class Ours: - def side(self) -> str: - return \"ours\" +class Boundary(StrEnum): + STRICT = auto() + CONFORM = auto() - def reset(self) -> object: - # `object`, because the outside base's `reset` answers `None` and a narrower - # annotation would make the difference a representation error rather than a value - return \"ours reset\" + def shout(self) -> str: + return self.name -class OursFirst(Ours, codecs.Codec): - def label(self) -> str: - return \"first\" +class Plain(StrEnum): + # no constant, so the namespace the metaclass is handed is the whole class body + def shout(self) -> str: + return \"plain\" -class OutsideFirst(codecs.StreamWriter, Ours): - # the outside base comes first, so *its* `reset` is what the mro reaches and ours is - # shadowed — a direct call on a receiver typed as `Ours` would answer with ours - def label(self) -> str: - return \"outside\" +FIRST = Boundary.STRICT +"; + let built = match build_source( + source, + "by_diff_metaconstant", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // nothing is declined: the class is emitted, and only the *construction* the runtime + // picks for it changes + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaconstant as m\n\ + print(m.Boundary._member_names_)\n\ + print(isinstance(m.FIRST, m.Boundary), m.FIRST is m.Boundary.STRICT)\n\ + print(m.Boundary.STRICT.shout(), m.Boundary('strict') is m.Boundary.STRICT)\n\ + print([c.__name__ for c in m.Boundary.__mro__])\n\ + print(type(m.Boundary).__name__, type(m.Plain).__name__)\n\ + print(m.Plain._member_names_, m.Plain().shout() if False else 'plain')\n\ + print(type(m.Boundary.shout).__name__, type(m.Plain.shout).__name__)\n", + ); + assert_eq!( + out, + "['STRICT', 'CONFORM']\n\ + True True\n\ + STRICT True\n\ + ['Boundary', 'StrEnum', 'str', 'ReprEnum', 'Enum', 'object']\n\ + EnumType EnumType\n\ + [] plain\n\ + function method_descriptor" + ); +} +#[test] +fn a_dunder_the_module_body_hangs_on_a_class_keeps_it_off_the_compiled_surface() { + // `ctypes` writes `c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte` under the class + // statement, and the adoption that carries a twin's attributes across leaves every + // dunder behind — a dunder is what a type slot answers, and a second answer sitting in + // the dict would disagree with it. so the attribute has nowhere to land and the class + // has to decline. + // + // the two classes are the two constructions: `Meta` can only be built by calling its + // metaclass and `Spec` comes from a type spec, and the adoption is the same for both. + // `plain` is the boundary in the other direction — a name that is not a dunder *is* + // carried, and is not a reason to turn anything down + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_hungdunder"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABCMeta -class AsDict(dict, Ours): - def label(self) -> str: - return \"dict\" +class Meta(metaclass=ABCMeta): + TAG = 1 -class AsInt(int, Ours): def label(self) -> str: - return \"int\" - - -class OurError(Exception): - def which(self) -> str: - return \"ourerror\" - + return \"meta\" -class Diamond(OurError, ValueError): - # the shared ancestor is outside: `Exception` is above both legs, and the mro has to - # place it once and after both - def label(self) -> str: - return \"diamond\" +class Spec: + TAG = 2 -class Under(OursFirst): - # a class of this module's own built on the mixture: the layout chain runs through - # a class that has none, so this one declares none either def label(self) -> str: - return \"under\" + return \"spec\" -def through_the_base(o: Ours) -> str: - return o.side() +class Untouched: + TAG = 3 - -def resetting(o: Ours) -> object: - return o.reset() + def label(self) -> str: + return \"untouched\" -def exactly(which: int) -> str: - if which == 0: - return OursFirst().side() - return OutsideFirst(None).side() -", - &[ - "[c.__name__ for c in m.OursFirst.__mro__]", - "[c.__name__ for c in m.OutsideFirst.__mro__]", - "[c.__name__ for c in m.Diamond.__mro__]", - "(m.OursFirst.__base__.__name__, m.AsDict.__base__.__name__, m.AsInt.__base__.__name__)", - "(m.OursFirst().side(), m.OursFirst().label())", - "(m.OutsideFirst(None).side(), m.OutsideFirst(None).label())", - "[m.through_the_base(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None), m.Under())]", - "[m.exactly(n) for n in (0, 1)]", - "[m.resetting(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None))]", - "([c.__name__ for c in m.Under.__mro__], m.Under().label(), m.Under().side())", - // the outside base still owns the instance it always owned - "(sorted(m.AsDict(a=1, b=2).items()), m.AsDict().label())", - "(int(m.AsInt(7)) + 1, m.AsInt(7).label(), m.AsInt(7).side())", - "(m.Diamond('boom').args, str(m.Diamond('boom')), m.Diamond('x').which(), m.Diamond('x').label())", - "(isinstance(m.Diamond('x'), ValueError), isinstance(m.Diamond('x'), m.OurError))", - // an instance of the mixture carries whatever `__dict__` the outside base - // brought: a class of ours alone has none, and losing it here silently - // turned every attribute read on one into a walk off the end of the object - "(lambda o: (setattr(o, 'kept', 3), o.kept, o.__dict__))(m.OursFirst())", - // and a python subclass of the result, which is a class statement on it - "(lambda C: (C().side(), C().label(), [c.__name__ for c in C.__mro__]))\ - (type('Py', (m.OursFirst,), {'label': lambda self: 'py'}))", - ], +Meta.__marker__ = Meta +Spec.__marker__ = Spec +Untouched.plain = 4 +"; + let built = match build_source( + source, + "by_diff_hungdunder", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "Meta", + "the module body writes `__marker__` onto `Meta`, which the emitted type does not carry" + ), + ( + "Spec", + "the module body writes `__marker__` onto `Spec`, which the emitted type does not carry" + ) + ] + ); + let out = run( + &python, + &dir, + "import by_diff_hungdunder as m\n\ + print(m.Meta.__marker__ is m.Meta, m.Spec.__marker__ is m.Spec, m.Untouched.plain)\n\ + print(m.Meta().label(), m.Spec().label(), m.Untouched().label())\n\ + print(type(m.Meta.label).__name__, type(m.Spec.label).__name__,\n\ + \x20 type(m.Untouched.label).__name__)\n", + ); + assert_eq!( + out, + "True True 4\n\ + meta spec untouched\n\ + function function method_descriptor" ); } #[test] -fn a_base_beside_an_outside_one_is_built_by_calling_its_metaclass() { - // a type spec takes its whole instance shape from the one base python picks out of - // the list. where that is a class of ours the `__dict__` an outside base needs is - // dropped, and the type then claims a managed dict it has no room for — the first - // attribute read on an instance walks off the object and segfaults. calling the - // metaclass works the shape out from every base at once +fn a_constant_that_reads_back_differently_every_time_is_not_turned_down_for_it() { + // a class-level constant is read out of the *mapping* the class body wrote rather + // than through a lookup on the class, and `__class_getitem__ = classmethod(f)` is + // where the two differ: a lookup runs the descriptor and answers a freshly bound + // method, and one bound to the interpreted definition at that. copying that is what + // used to make `Holder[int]` answer the twin instead of `Holder`, and it made the + // check after the metaclass call — which compares the finished class against the + // values it was handed — turn down every class with such a constant, since no two + // reads are the same object. almost every container type in the stdlib has one. // - // the descriptors say which build answered: a class that fell back to its - // interpreted definition would agree on every value above and say `function` + // `Spec` is the boundary: both constructions carry the classmethod itself now, so + // both answer the class the interpreter answers let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_mixedmeta"); + let dir = std::env::temp_dir().join("by_diff_metaunstable"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -import abc -import codecs +from abc import ABCMeta -class Ours: - def side(self) -> str: - return \"ours\" +def _get(cls, item): + return cls -class Mixed(Ours, codecs.Codec): +class Holder(metaclass=ABCMeta): + __class_getitem__ = classmethod(_get) + def label(self) -> str: - return \"mixed\" + return \"holder\" -class Metaclassed(Ours, abc.ABC): - # a base whose metaclass is not `type` rules the spec out on its own, and the - # keyword the header carries has nowhere to go in one either +class Spec: + __class_getitem__ = classmethod(_get) + def label(self) -> str: - return \"meta\" + return \"spec\" "; let built = match build_source( source, - "by_diff_mixedmeta", + "by_diff_metaunstable", &toolchain, &dir, &Options { @@ -6538,209 +8746,282 @@ class Metaclassed(Ours, abc.ABC): let out = run( &python, &dir, - "import by_diff_mixedmeta as m\n\ - print(type(m.Mixed.label).__name__, type(m.Ours.side).__name__)\n\ - print(m.Mixed.__base__.__name__, type(m.Metaclassed).__name__)\n\ - o = m.Mixed()\n\ - o.kept = 4\n\ - print(o.side(), o.label(), o.kept, o.__dict__)\n", + "import by_diff_metaunstable as m\n\ + print(m.Holder().label(), m.Spec().label())\n\ + print(type(m.Holder.label).__name__, type(m.Spec.label).__name__)\n\ + print(m.Holder[int] is m.Holder, m.Spec[int] is m.Spec)\n\ + print(m.Holder[int] is m.Spec[int])\n", ); assert_eq!( out, - "method_descriptor method_descriptor\n\ - Ours ABCMeta\n\ - ours mixed 4 {'kept': 4}" + "holder spec\n\ + method_descriptor method_descriptor\n\ + True True\n\ + False" ); } #[test] -fn a_class_built_through_its_metaclass_agrees() { - // `PyType_FromSpecWithBases` gives the type it builds `type` for a metaclass, so it - // cannot be handed a base with any other one, and `PyType_FromMetaclass` refuses a - // metaclass that overrides `__new__` — which `ABCMeta` does. so the type is built - // the way python builds one: by calling the metaclass with a namespace. +fn a_metaclass_that_raises_on_the_namespace_it_is_handed_leaves_the_import_standing() { + // `ssl.Purpose`'s shape, and the one thing worse than a wrong answer: `EnumType` is + // handed a namespace whose members are the twin's *finished* ones and tries to build a + // member out of a member — `Obj.__new__(cls, a, b)` against a `__new__` that takes one + // argument. that raises before the check after the call could turn the class down, and + // propagating it fails `by_exec` and takes the whole import with it. // - // the methods go *in* that namespace rather than onto the finished type, which is - // what fills the type slots — `type.__new__` runs the same fixup a class statement - // does, so `__repr__` becomes `tp_repr` with no adapter of ours - agree( - "metaclass", - "\ -from abc import ABC, ABCMeta - - -class Node(ABC): - def kind(self) -> str: - return \"node\" - - def __repr__(self) -> str: - return \"Node(\" + self.kind() + \")\" - - def __len__(self) -> int: - return 3 - + // the interpreted definition already built this class — the fallback source ran first — + // so the raise says the reconstruction is wrong, not that the class is unbuildable. it + // is the same refusal the check makes, reached earlier. `Plain` is the boundary: the + // same metaclass with a namespace it can work with still reaches the compiled type + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaraise"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from collections import namedtuple +from enum import Enum -class Deeper(Node): - # a base of *ours* whose own base is not: the layout is nobody's, and the metaclass - # is inherited down the chain - def kind(self) -> str: - return \"deeper\" +class Obj(namedtuple(\"Obj\", \"a b\")): + __slots__ = () -class Keyed(metaclass=ABCMeta): - # no bases at all, only the keyword — python supplies `(object,)` itself - def kind(self) -> str: - return \"keyed\" + def __new__(cls, text): + return super().__new__(cls, text, text.upper()) -class Checked(ABC): - # an `__init__` of its own that stores nothing: it has to reach `tp_init`, which on - # this construction is python's dispatcher reading the namespace entry - def __init__(self, value: int) -> None: - if value < 0: - raise ValueError(\"negative\") +class Kind(Obj, Enum): + ONE = \"one\" + TWO = \"two\" -class Equated(ABC): - # a class defining `__eq__` and not `__hash__` is unhashable. a spec has to be told - # that; `type.__new__` does it for itself, so this is the same answer reached two - # different ways depending on which construction the bases allowed - def __eq__(self, other: object) -> bool: - return isinstance(other, Equated) +class Plain(Enum): + def shout(self) -> str: + return \"plain\" -def widened(value: object) -> str: - if isinstance(value, Node): - return \"node \" + value.kind() - return \"other\" -", - &[ - // the metaclass is the one python would have worked out, not `type` - "(type(m.Node).__name__, type(m.Keyed).__name__, type(m.Deeper).__name__)", - // and the whole ancestry with it - "[c.__name__ for c in m.Node.__mro__]", - "[c.__name__ for c in m.Keyed.__mro__]", - "[c.__name__ for c in m.Deeper.__mro__]", - // an abstract base answers for it, which is the point of the metaclass - "(isinstance(m.Node(), __import__('abc').ABC), issubclass(m.Node, __import__('abc').ABC))", - "(isinstance(m.Deeper(), m.Node), issubclass(m.Deeper, m.Node))", - // `ABCMeta.__new__` ran, and ran over *this* class's namespace - "sorted(m.Node.__abstractmethods__)", - // a dunder written in the class body fills its slot - "repr(m.Node())", - "repr(m.Deeper())", - "len(m.Node())", - // and the ordinary methods still bind - "(m.Node().kind(), m.Deeper().kind(), m.Keyed().kind())", - "[m.widened(v) for v in (m.Node(), m.Deeper(), 1)]", - // a written `__init__` runs, and the errors around it are python's own - "(m.Checked(1) is not None, type(_capture(m.Checked, -1)).__name__)", - "type(_capture(m.Checked)).__name__", - // and `__eq__` without `__hash__` still takes the hash away - "(m.Equated() == m.Equated(), m.Equated() == 1)", - "type(_capture(hash, m.Equated())).__name__", - // the class reports the module it was written in, not `builtins` - "(m.Node.__module__, m.Keyed.__module__)", - "(m.Node.__name__, m.Node.__qualname__)", - // an imported base reaches this module as a lazy proxy, so `__mro_entries__` - // is what turns it into the class — and python records what was written - "repr(m.Node.__orig_bases__)", - ], +FIRST = Kind.ONE +"; + let built = match build_source( + source, + "by_diff_metaraise", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // `Obj` names a call as its base, which is a decline of its own and not this one + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + ["Obj"] + ); + let out = run( + &python, + &dir, + "import by_diff_metaraise as m\n\ + print(m.Kind.ONE.a, m.Kind.ONE.b, m.FIRST is m.Kind.ONE)\n\ + print(isinstance(m.FIRST, m.Kind), m.Kind(m.Kind.ONE.value) is m.FIRST)\n\ + print(m.Kind._member_names_, type(m.Kind).__name__)\n\ + print(type(m.Plain.shout).__name__)\n", + ); + assert_eq!( + out, + "one ONE True\n\ + True True\n\ + ['ONE', 'TWO'] EnumType\n\ + method_descriptor" ); } #[test] -fn a_class_keyword_reaches_init_subclass_agrees() { - // a keyword is the metaclass's business, and a type spec has nowhere to put one. - // the base is built at runtime because a class this module *emits* would have its - // layout laid out here, which the metaclass construction gives up - agree( - "classkeyword", - "\ -from abc import ABCMeta - +fn a_class_level_constant_beside_a_base_of_ours_reaches_the_metaclass_namespace() { + // the shape 69 of the stdlib's 84 instances of the old constant decline had: no class + // keyword anywhere, a base this module emits standing beside one from outside, and a + // constant. a spec cannot work that base list out, so the metaclass builds it — and + // the constant goes into the namespace it is handed rather than onto the type + // afterwards, which is what makes that construction answer for such a class at all. + // + // `issubclass` is what the decline used to be protecting: with `Reader` interpreted + // and `Codec` emitted it answers False where python answers True, so the two had to + // decline together. both are compiled here, and `method_descriptor` against + // `function` is what says so — `Alone` is the boundary that never needed the base + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaconstant_base"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import codecs -def _tagged(cls: type, tag: str = \"none\", ready: bool = False, level: int = 0) -> None: - cls.tag = tag - cls.ready = ready - cls.level = level +class Codec(codecs.Codec): + def label(self) -> str: + return \"codec\" -Base = ABCMeta(\"Base\", (), {\"__init_subclass__\": classmethod(_tagged)}) +class Reader(Codec, codecs.StreamReader): + tag = 1 -class Alpha(Base, tag=\"alpha\"): def kind(self) -> str: - return \"alpha\" + return \"reader\" -class Beta(Base, tag=\"beta\", ready=True, level=2): - # a literal keyword as well as a name: both are evaluated where a class body would - # have evaluated them +class Alone(codecs.Codec): + tag = 2 + def kind(self) -> str: - return \"beta\" + return \"alone\" +"; + let built = match build_source( + source, + "by_diff_metaconstant_base", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!(declined, Vec::new()); + let out = run( + &python, + &dir, + "import by_diff_metaconstant_base as m\n\ + print(issubclass(m.Reader, m.Codec), m.Reader.tag, m.Alone.tag)\n\ + print([c.__name__ for c in m.Reader.__mro__])\n\ + print(type(m.Codec.label).__name__, type(m.Reader.kind).__name__,\n\ + \x20 type(m.Alone.kind).__name__)\n", + ); + assert_eq!( + out, + "True 1 2\n\ + ['Reader', 'Codec', 'StreamReader', 'Codec', 'object']\n\ + method_descriptor method_descriptor method_descriptor" + ); +} +#[test] +fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { + // `__slots__` is the constant that proves the namespace is where these have to go. + // `type.__new__` reads it *out of the namespace* to decide whether the instances get + // a dict at all, so one copied onto the finished type afterwards is not a `__slots__` + // — the class already has the dict, and the entry sits there saying otherwise. + // + // 29 stdlib classes are this shape. `Open` is the boundary: the same base and the + // same construction with no `__slots__`, and python gives *its* instances a dict, so + // this is not a rule about emitted classes but about what the body wrote + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaslots"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABC -# the keyword's own text is arbitrary, and the C string form of it stopped at a NUL -class Gamma(Base, tag=\"a\\x00b\"): - def kind(self) -> str: - return \"gamma\" -", - &[ - // the keyword reached `__init_subclass__`, which the metaclass's `__new__` - // is what calls - "(m.Alpha.tag, m.Alpha.ready, m.Alpha.level)", - "(m.Beta.tag, m.Beta.ready, m.Beta.level)", - "ascii(m.Gamma.tag)", - "(m.Alpha().kind(), m.Beta().kind(), m.Gamma().kind())", - // and the metaclass itself came from the base - "(type(m.Alpha).__name__, type(m.Beta).__name__)", - "[c.__name__ for c in m.Beta.__mro__]", - "sorted(m.Beta.__abstractmethods__)", - ], + +class Slotted(ABC): + __slots__ = () + + def label(self) -> str: + return \"slotted\" + + +class Open(ABC): + def label(self) -> str: + return \"open\" +"; + let built = match build_source( + source, + "by_diff_metaslots", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaslots as m\n\ + print(m.Slotted.__slots__, hasattr(m.Slotted(), '__dict__'), hasattr(m.Open(), '__dict__'))\n\ + print(m.Slotted().label(), m.Open().label())\n\ + print(type(m.Slotted.label).__name__, type(m.Open.label).__name__)\n", + ); + assert_eq!( + out, + "() False True\n\ + slotted open\n\ + method_descriptor method_descriptor" ); } #[test] -fn which_build_answers_for_a_metaclass_class() { - // the two `agree` tests above cannot see this on their own: a class that quietly - // fell back to its interpreted definition answers *identically*, so both would pass - // on a compiler that built nothing at all. this one names the build — a compiled - // method is a descriptor, an interpreted one is a plain function. +fn a_class_constant_naming_another_class_reaches_the_metaclass_namespace_remapped() { + // the value a constant carries comes off the twin, so `pair = Other` in a class body + // hands over the *interpreted* `Other` — a class nothing else in the module can + // reach, and one `isinstance` denies against `m.Other`. the substitution that fixes + // that is the copy's, and the namespace has to make the same one or the two + // constructions disagree about what a constant is. // - // it is also where the restriction is pinned down. the construction through a - // metaclass hands back a type whose instance layout is the metaclass's answer, so a - // class appending fields to its base has nowhere to put them and stays interpreted; - // its fieldless sibling, on the same base, does not + // `Below` is what forces the metaclass here: `ABCMeta` closes the spec, so the + // constant goes in before the call rather than onto the type after it let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_metafields"); + let dir = std::env::temp_dir().join("by_diff_metaconstant_remap"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from abc import ABC, ABCMeta - - -class Fielded(ABC): - def __init__(self, value: int) -> None: - self.value = value +from abc import ABCMeta - def doubled(self) -> int: - return self.value * 2 +class Other: + def kind(self) -> str: + return \"other\" -class Fieldless(ABC): - def label(self) -> str: - return \"fieldless\" +class Below(metaclass=ABCMeta): + pair = Other -class Keyed(metaclass=ABCMeta): def label(self) -> str: - return \"keyed\" + return \"below\" "; let built = match build_source( source, - "by_diff_metafields", + "by_diff_metaconstant_remap", &toolchain, &dir, &Options { @@ -6755,80 +9036,61 @@ class Keyed(metaclass=ABCMeta): return; } }; - let declined: Vec<(&str, &str)> = built - .declined - .iter() - .map(|declined| (declined.name.as_str(), declined.reason.as_str())) - .collect(); - assert_eq!( - declined, - vec![( - "Fielded", - "a class with fields of its own needs a base whose metaclass is `type`" - )] - ); + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import by_diff_metafields as m\n\ - print(m.Fielded(4).doubled(), m.Fielded(4).value)\n\ - print(m.Fieldless().label(), m.Keyed().label())\n\ - print(type(m.Fielded).__name__, type(m.Fieldless).__name__, type(m.Keyed).__name__)\n\ - print(isinstance(m.Fielded(1), m.Fielded), isinstance(m.Fieldless(), m.Fieldless))\n\ - print(type(m.Fielded.doubled).__name__, type(m.Fieldless.label).__name__,\n\ - \x20 type(m.Keyed.label).__name__)\n", + "import by_diff_metaconstant_remap as m\n\ + print(m.Below.pair is m.Other, isinstance(m.Below.pair(), m.Other))\n\ + print(m.Below.pair().kind(), m.Below().label())\n\ + print(type(m.Below.label).__name__, type(m.Other.kind).__name__)\n", ); assert_eq!( out, - "8 4\n\ - fieldless keyed\n\ - ABCMeta ABCMeta ABCMeta\n\ - True True\n\ - function method_descriptor method_descriptor" + "True True\n\ + other below\n\ + method_descriptor method_descriptor" ); } #[test] -fn a_class_level_constant_keeps_its_class_off_the_metaclass_construction() { - // a class-level constant is copied onto the *finished* type, and through a metaclass - // that is too late: the metaclass has already decided what the class defines from a - // namespace the constant was never in. an `EnumType` handed a memberless namespace - // declares no members, and the copy then lands them in the type's dict behind its - // back — `Boundary.STRICT` answers while `_member_names_` is empty and - // `isinstance(FIRST, Boundary)` is False. that was a silent wrong answer, so a class - // with any constant keeps the interpreted definition instead. +fn a_class_the_module_pops_out_of_its_own_globals_stays_off_the_compiled_surface() { + // `ast` builds `Num` and then pops the name straight out of its own globals. that is + // a `del` whose target this cannot read — the name comes off a comprehension there — + // so every definition the module writes is treated as one the pop could have taken. + // installing a compiled `Gone` over a name the body removed would put a class on the + // surface python does not have there, and the interpreted definition the construction + // would otherwise fall back to is not there to be found either. // - // the two classes are the boundary: same base, same metaclass, and only `Boundary` - // has a constant. `function` against `method_descriptor` is what says the fallback is - // exactly that narrow — widening the gate would make `Plain` say `function` too + // the class-level-constant gate used to carry this, and this is what stayed behind + // when it went. `Kept` is no longer a boundary — the rule reaches the whole module, + // which is what its second decline says let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_metaconstant"); + let dir = std::env::temp_dir().join("by_diff_poppedclass"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from enum import StrEnum, auto +from abc import ABCMeta -class Boundary(StrEnum): - STRICT = auto() - CONFORM = auto() +class Gone(metaclass=ABCMeta): + TAG = 1 - def shout(self) -> str: - return self.name + def label(self) -> str: + return \"gone\" -class Plain(StrEnum): - # no constant, so the namespace the metaclass is handed is the whole class body - def shout(self) -> str: - return \"plain\" +class Kept: + def label(self) -> str: + return \"kept\" -FIRST = Boundary.STRICT +HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} "; let built = match build_source( source, - "by_diff_metaconstant", + "by_diff_poppedclass", &toolchain, &dir, &Options { @@ -6843,30 +9105,37 @@ FIRST = Boundary.STRICT return; } }; - // nothing is declined: the class is emitted, and only the *construction* the runtime - // picks for it changes - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "Gone", + "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" + ), + ( + "Kept", + "`Kept` is rebound at module level, so installing this over it would replace what the rebind produced" + ) + ] + ); let out = run( &python, &dir, - "import by_diff_metaconstant as m\n\ - print(m.Boundary._member_names_)\n\ - print(isinstance(m.FIRST, m.Boundary), m.FIRST is m.Boundary.STRICT)\n\ - print(m.Boundary.STRICT.shout(), m.Boundary('strict') is m.Boundary.STRICT)\n\ - print([c.__name__ for c in m.Boundary.__mro__])\n\ - print(type(m.Boundary).__name__, type(m.Plain).__name__)\n\ - print(m.Plain._member_names_, m.Plain().shout() if False else 'plain')\n\ - print(type(m.Boundary.shout).__name__, type(m.Plain.shout).__name__)\n", + "import by_diff_poppedclass as m\n\ + print('Gone' in m.__dict__, 'Kept' in m.__dict__)\n\ + print(m.HIDDEN['Gone'].TAG, m.HIDDEN['Gone']().label(), m.Kept().label())\n\ + print(type(m.HIDDEN['Gone'].label).__name__, type(m.Kept.label).__name__)\n", ); assert_eq!( out, - "['STRICT', 'CONFORM']\n\ - True True\n\ - STRICT True\n\ - ['Boundary', 'StrEnum', 'str', 'ReprEnum', 'Enum', 'object']\n\ - EnumType EnumType\n\ - [] plain\n\ - function method_descriptor" + "False True\n\ + 1 gone kept\n\ + function function" ); } @@ -7030,9 +9299,12 @@ fn a_late_gift_that_could_hand_the_interpreted_class_back_is_left_alone() { // // so `SAMPLE` (an instance of the interpreted class), `ITEMS` (a list, which can be // given one after the question is asked) and `HIDDEN` (a tuple holding one) do not - // come across, while `MARKER`, `PAIR` and `shout` do. a dunder never does: a name in - // the type's dict does not fill a type slot, so `__ge__` there would answer - // `a.__ge__(b)` while `a >= b` still went to the slot. + // come across, while `MARKER`, `PAIR` and `shout` do. + // + // a dunder never comes across either — a name in the type's dict does not fill a type + // slot, so `__ge__` there would answer `a.__ge__(b)` while `a >= b` still went to the + // slot — and that is why a class the body hangs one on is turned down instead. `Ordered` + // is that half: dropping the entry is as wrong an answer as carrying it would be. // // `method_descriptor` is what says the compiled type answered at all: a class that // fell back to its interpreted definition would carry every one of these, because it @@ -7053,13 +9325,18 @@ class Held: return \"held\" +class Ordered: + def tag(self) -> str: + return \"ordered\" + + Held.MARKER = 3 Held.PAIR = Other Held.SAMPLE = Other() Held.ITEMS = [1, 2] Held.HIDDEN = (Other,) -Held.__ge__ = lambda self, right: True Held.shout = lambda self: self.tag().upper() +Ordered.__ge__ = lambda self, right: True "; let built = match build_source( source, @@ -7078,20 +9355,30 @@ Held.shout = lambda self: self.tag().upper() return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + ["Ordered"] + ); let out = run( &python, &dir, "import by_diff_twinshapes as m\n\ print(m.Held.MARKER, m.Held.PAIR is m.Other, m.Held().shout())\n\ print(hasattr(m.Held, 'SAMPLE'), hasattr(m.Held, 'ITEMS'), hasattr(m.Held, 'HIDDEN'))\n\ - print(type(m.Held.tag).__name__, type(m.Other.tag).__name__)\n", + print(m.Ordered() >= m.Ordered(), m.Ordered().tag())\n\ + print(type(m.Held.tag).__name__, type(m.Other.tag).__name__,\n\ + \x20 type(m.Ordered.tag).__name__)\n", ); assert_eq!( out, "3 True HELD\n\ False False False\n\ - method_descriptor method_descriptor" + True ordered\n\ + method_descriptor method_descriptor function" ); } @@ -7403,17 +9690,19 @@ class Plain: } #[test] -fn a_subclass_of_a_class_the_metaclass_gates_turn_down_is_built_here() { - // both metaclass gates are asked while the layouts settle, so a class either of them - // turns down leaves the layout set — and its subclass is then laid out on the - // interpreted definition the way every other declining class's subclass is. asked - // while the body was lowered instead, the base stayed in the set and each subclass - // cascaded behind it, which is what this pair of declines used to be four of. +fn a_subclass_of_a_class_the_metaclass_gate_turns_down_is_built_here() { + // the metaclass gate is asked while the layouts settle, so a class it turns down + // leaves the layout set — and its subclass is then laid out on the interpreted + // definition the way every other declining class's subclass is. asked while the body + // was lowered instead, the base stayed in the set and the subclass cascaded behind + // it, which is what this decline used to be two of. // - // the bases here carry `ABCMeta`, so `PyType_FromSpecWithBases` is closed to the - // subclass and its metaclass builds it — `method_descriptor` is what says that - // construction happened at all, since a subclass that fell back would answer every - // value here from a `function` + // `Constant` is the other half, and it is the boundary: a class-level constant no + // longer turns a class down, so both it and its subclass are built here. the bases + // carry `ABCMeta`, so `PyType_FromSpecWithBases` is closed to every class in this + // module and the metaclass builds them — `method_descriptor` is what says that + // construction happened at all, since a class that fell back would answer from a + // `function` let Some((python, toolchain)) = environment() else { return; }; @@ -7462,7 +9751,7 @@ class BelowConstant(Constant): return; } }; - // exactly the two bases, and nothing behind them: a subclass in this list is the + // exactly the one base, and nothing behind it: a subclass in this list is the // cascade this move exists to stop assert_eq!( built @@ -7470,7 +9759,7 @@ class BelowConstant(Constant): .iter() .map(|declined| declined.name.as_str()) .collect::>(), - ["Decorated", "Constant"] + ["Decorated"] ); let out = run( &python, @@ -7480,7 +9769,8 @@ class BelowConstant(Constant): print(m.BelowConstant().size(), m.BelowConstant.TAG, m.BelowConstant().label())\n\ print([b.__name__ for b in m.BelowConstant.__mro__])\n\ print(isinstance(m.BelowConstant(), m.Constant), isinstance(m.BelowDecorated(), m.Decorated))\n\ - print(type(m.BelowDecorated.size).__name__, type(m.BelowConstant.size).__name__)\n", + print(type(m.BelowDecorated.size).__name__, type(m.BelowConstant.size).__name__,\n\ + \x20 type(m.Constant.label).__name__)\n", ); assert_eq!( out, @@ -7488,22 +9778,22 @@ class BelowConstant(Constant): 2 1 constant\n\ ['BelowConstant', 'Constant', 'object']\n\ True True\n\ - method_descriptor method_descriptor" + method_descriptor method_descriptor method_descriptor" ); } #[test] -fn a_pair_the_body_cross_links_agrees_when_a_gate_took_their_base_out_of_the_layouts() { - // `urllib.parse`'s shape, and the one that reverted this move the first time: the - // base declines at the class-level-constant gate, so both result classes are built - // here — over an interpreted base whose metaclass is `type`, which means a real - // emitted type replaces each twin in the namespace. `_pair` then runs against the - // twins, because the whole module body runs before module init installs anything. +fn a_pair_the_body_cross_links_agrees_when_the_link_is_made_after_the_class_statement() { + // `urllib.parse`'s shape, and the one that reverted this move the first time: `_pair` + // runs against the twins, because the whole module body runs before module init + // installs anything, and it hangs each result class off the other under a name no + // class body wrote. // // what makes it agree is that an emitted type carries what the body gave its twin // *and* remaps a carried twin to the type standing in for it — without the remap // `Text._encoded_counterpart()` builds something `isinstance` says is not a - // `m.Bytes` + // `m.Bytes`. the base used to decline at the class-level-constant gate and take the + // whole chain with it; it is emitted now, which is what the empty decline list says let Some((python, toolchain)) = environment() else { return; }; @@ -7563,15 +9853,15 @@ _pair() return; } }; - // `Root` and `Extra` follow `Mixin` down because an emitted class cannot have an - // interpreted subclass. `Text` and `Bytes` are the two that must not be here + // `Mixin` used to decline and take `Root` and `Extra` down with it, because an + // emitted class cannot have an interpreted subclass. none of the five is here now assert_eq!( built .declined .iter() .map(|declined| declined.name.as_str()) .collect::>(), - ["Mixin", "Root", "Extra"] + Vec::<&str>::new() ); let out = run( &python, @@ -7594,41 +9884,175 @@ _pair() } #[test] -fn a_private_name_in_a_class_body_is_mangled() { - // python binds an identifier of two leading underscores written in `class C` as - // `_C__spam`, whatever it names. the compiler read the written name, so the compiled - // class published `__read` and `__buffer` where python publishes `_Printer__read` - // and `_Printer__buffer` — `symtable.Function` lost five class attributes and three - // methods that way, and every read of one inside a method raised `AttributeError` - // because the name it looked for was never bound +fn a_private_name_in_a_class_body_is_mangled() { + // python binds an identifier of two leading underscores written in `class C` as + // `_C__spam`, whatever it names. the compiler read the written name, so the compiled + // class published `__read` and `__buffer` where python publishes `_Printer__read` + // and `_Printer__buffer` — `symtable.Function` lost five class attributes and three + // methods that way, and every read of one inside a method raised `AttributeError` + // because the name it looked for was never bound + // + // `_Printer` also exercises the class's *own* leading underscore, which the mangling + // strips + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_privatemangle"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class _Printer: + __LIMIT = 4 + + def __init__(self, source: str) -> None: + self.__buffer = source + self.plain = 0 + + def __read(self) -> str: + return self.__buffer + + def take(self) -> str: + return self.__read() + + def limit(self) -> int: + return self.__LIMIT +"; + let built = match build_source( + source, + "by_diff_privatemangle", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_privatemangle as m\n\ + p = m._Printer('hi')\n\ + print(p.take(), p.limit(), getattr(p, '_Printer__buffer'))\n\ + print(hasattr(p, '__buffer'), hasattr(m._Printer, '__read'))\n\ + print(m._Printer._Printer__LIMIT, callable(m._Printer._Printer__read))\n\ + print(type(m._Printer.take).__name__)\n", + ); + assert_eq!( + out, + "hi 4 hi\n\ + False False\n\ + 4 True\n\ + method_descriptor" + ); +} + +#[test] +fn a_name_that_is_both_a_class_constant_and_a_field_keeps_the_interpreted_class() { + // the constant is copied into the type's dict *after* `PyType_Ready` put the field's + // descriptor there, so the constant wins and every instance answers the class-level + // value instead of its own. that is a silent wrong answer, so the class declines and + // the interpreted definition — which python's own rules already get right — answers + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_constantfieldclash"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Tagged: + KIND: str = \"class-level\" + + def __init__(self, kind: str) -> None: + self.KIND = kind + + def read(self) -> str: + return self.KIND +"; + let built = match build_source( + source, + "by_diff_constantfieldclash", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built.declined.iter().any(|declined| declined + .reason + .contains("both a class-level constant and a field")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_constantfieldclash as m\n\ + print(m.Tagged('mine').KIND, m.Tagged('mine').read(), m.Tagged.KIND)\n\ + print(type(m.Tagged.read).__name__)\n", + ); + assert_eq!(out, "mine mine class-level\nfunction"); +} + +#[test] +fn a_decorated_class_carries_the_body_its_own_decorator_was_handed() { + // a class-level constant is copied off the body the interpreted `class` statement + // wrote, taken while that statement runs and before any of the class's decorators is + // handed it. reading the finished definition instead is what this used to do, and by + // then the decorator has been over it. // - // `_Printer` also exercises the class's *own* leading underscore, which the mangling - // strips + // `@dataclass` shows both halves of what that cost. `later` has no default, so + // `_process_class` *deletes* the `field(init=False)` and left nothing to copy — the + // emitted type's annotation then read as a required argument after a defaulted one, + // and the module raised `TypeError` at import. `hidden` has one, so the `Field` was + // replaced by the bare `2` and `repr=False` went with it — the emitted `repr` showed + // a field the interpreted one hides, which no sweep can see. + // + // the descriptors are what say the compiled types answered: an interpreted leg has a + // plain `function` in both places let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_privatemangle"); + let dir = std::env::temp_dir().join("by_diff_decoratedconstant"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class _Printer: - __LIMIT = 4 +from dataclasses import dataclass, field - def __init__(self, source: str) -> None: - self.__buffer = source - self.plain = 0 - def __read(self) -> str: - return self.__buffer +@dataclass +class Sub: + tag: str = \"b\" + later: list[str] = field(init=False) - def take(self) -> str: - return self.__read() + def name(self) -> str: + return self.tag - def limit(self) -> int: - return self.__LIMIT + +@dataclass +class Holder: + shown: int = 1 + hidden: int = field(default=2, repr=False) + + def total(self) -> int: + return self.shown + self.hidden "; let built = match build_source( source, - "by_diff_privatemangle", + "by_diff_decoratedconstant", &toolchain, &dir, &Options { @@ -7647,46 +10071,63 @@ class _Printer: let out = run( &python, &dir, - "import by_diff_privatemangle as m\n\ - p = m._Printer('hi')\n\ - print(p.take(), p.limit(), getattr(p, '_Printer__buffer'))\n\ - print(hasattr(p, '__buffer'), hasattr(m._Printer, '__read'))\n\ - print(m._Printer._Printer__LIMIT, callable(m._Printer._Printer__read))\n\ - print(type(m._Printer.take).__name__)\n", + "from dataclasses import fields\n\ + import by_diff_decoratedconstant as m\n\ + print(m.Sub('x').name(), repr(m.Holder()), m.Holder().total())\n\ + print([(f.name, f.init, f.repr) for f in fields(m.Sub)])\n\ + print([(f.name, f.init, f.repr) for f in fields(m.Holder)])\n\ + print(type(m.Sub.name).__name__, type(m.Holder.total).__name__)\n", ); assert_eq!( out, - "hi 4 hi\n\ - False False\n\ - 4 True\n\ - method_descriptor" + "x Holder(shown=1) 3\n\ + [('tag', True, True), ('later', False, True)]\n\ + [('shown', True, True), ('hidden', True, False)]\n\ + method_descriptor method_descriptor" ); } #[test] -fn a_name_that_is_both_a_class_constant_and_a_field_keeps_the_interpreted_class() { - // the constant is copied into the type's dict *after* `PyType_Ready` put the field's - // descriptor there, so the constant wins and every instance answers the class-level - // value instead of its own. that is a silent wrong answer, so the class declines and - // the interpreted definition — which python's own rules already get right — answers +fn the_class_body_capture_reaches_only_this_module_s_own_body() { + // the capture is a copy of the builtins mapping, carrying a `__build_class__` of ours, + // put in this module's dict for the length of the fallback run — so no other module and + // no other thread can reach it, which on a free-threaded interpreter is the difference + // between a scoped trick and a race. + // + // python gives a function the builtins its defining frame had, though, so every + // function the body defined holds that copy for as long as it lives. `make_held` is + // that function: calling it once the import is over must make an ordinary class, with + // nothing recorded into a mapping that has been released by then. + // + // it is also a class named `Held`, the same as the module-level one, and the body calls + // it before the module is finished. that inner one must not stand in for the outer: + // `EARLY` is the interpreted answer taken during the body, `Held.kind` is what the + // emitted type carried, and they say different things let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_constantfieldclash"); + let dir = std::env::temp_dir().join("by_diff_capturescope"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Tagged: - KIND: str = \"class-level\" +class Held: + kind = \"module\" - def __init__(self, kind: str) -> None: - self.KIND = kind + def where(self) -> str: + return \"outer\" - def read(self) -> str: - return self.KIND + +def make_held(): + class Held: + kind = \"local\" + + return Held + + +EARLY = make_held().kind "; let built = match build_source( source, - "by_diff_constantfieldclash", + "by_diff_capturescope", &toolchain, &dir, &Options { @@ -7701,21 +10142,27 @@ class Tagged: return; } }; + // `make_held` writes a class in a function body, which nothing lowers. the `Held` the + // module itself writes is the one that has to be emitted assert!( - built.declined.iter().any(|declined| declined - .reason - .contains("both a class-level constant and a field")), + !built + .declined + .iter() + .any(|declined| declined.name == "Held"), "declined: {:?}", built.declined ); let out = run( &python, &dir, - "import by_diff_constantfieldclash as m\n\ - print(m.Tagged('mine').KIND, m.Tagged('mine').read(), m.Tagged.KIND)\n\ - print(type(m.Tagged.read).__name__)\n", + "import builtins\n\ + import by_diff_capturescope as m\n\ + print(m.Held.kind, m.EARLY, m.make_held().kind)\n\ + print(type(m.Held.where).__name__)\n\ + print(m.__builtins__ is builtins.__dict__)\n\ + print(m.make_held.__builtins__['__build_class__'] is builtins.__build_class__)\n", ); - assert_eq!(out, "mine mine class-level\nfunction"); + assert_eq!(out, "module local local\nmethod_descriptor\nTrue\nTrue"); } #[test] @@ -8023,7 +10470,12 @@ fn a_class_whose_base_declined_declines_with_it() { // a class was emitted with its base *silently dropped* when that base declined: // codegen looked the base up among the emitted classes, found nothing, and built a // type with no bases at all. the subclass then lost everything the base brought, - // which is a wrong answer rather than a slow one + // which is a wrong answer rather than a slow one. + // + // `Inner` keeps its base by naming it: it stores nothing of its own, so it is built + // on whatever the name holds at import — the interpreted `Outer` — exactly as a + // class over a base out of this module is. `method_descriptor` against `function` + // is what says which of the two answered let Some((python, toolchain)) = environment() else { return; }; @@ -8067,19 +10519,23 @@ class Inner(Outer): declined.sort_unstable(); assert_eq!( declined, - vec![ - ("Inner", "`Outer` declined, so it is not a base to build on"), - ("Outer", "`__new__` fills a type slot with no adapter yet"), - ] + vec![("Outer", "`__new__` fills a type slot with no adapter yet")] ); let out = run( &python, &dir, "import by_diff_declinedbase as m\n\ print(issubclass(m.Inner, m.Outer), isinstance(m.Inner(), m.Outer))\n\ - print(m.Inner().label(), m.Inner().tag())\n", + print(m.Inner().label(), m.Inner().tag())\n\ + print(type(m.Inner.__dict__['tag']).__name__,\n\ + \x20 type(m.Outer.__dict__['label']).__name__)\n", + ); + assert_eq!( + out, + "True True\n\ + outer inner\n\ + method_descriptor function" ); - assert_eq!(out, "True True\nouter inner"); } #[test] @@ -8482,6 +10938,9 @@ class Listc(A): assert_eq!( declined, vec![ + // `A` is not here on its own account any more: its `@classmethod` carries + // its convention on the method table entry, and it reads no slot zero. it + // is at the end of the list instead, behind the subclasses that declined ( "Cm", "`classmethod` and `staticmethod` both leave something other than the receiver in slot zero" @@ -8522,9 +10981,9 @@ class Listc(A): "Listc", "a `super()` in a comprehension reads that comprehension's own frame, which only python 3.12 and later fold into the method's" ), - // `A` is the base every class above extends, and `Cm` is the first of them - // to decline — so the base goes interpreted with it, or `Cm`'s interpreted - // definition would be subclassing a compiled type that refuses to be a base + // every one of those is a class the fallback leaves interpreted, and each + // extends `A` — so an emitted `A` would be a base with interpreted + // subclasses, which a static type refuses to be ( "A", "`Cm` declined, so it extends the interpreted definition rather than this type" @@ -8533,11 +10992,7 @@ class Listc(A): ); // a declined class still answers, through the interpreted definition the fallback // left behind. only the two whose answer does not turn on the interpreter version - // are called: python raises in most of the rest, and differently across versions. - // - // `Cm->Acgo` is also what says the decline was needed — a compiled `classmethod` - // is a method descriptor behind one, which raises `TypeError` when a *type* is - // what reaches it + // are called: python raises in most of the rest, and differently across versions let out = run( &python, &dir, @@ -10115,6 +12570,90 @@ def free(x: int, step: int = 1) -> int: ); } +#[test] +fn an_arity_error_is_worded_by_the_interpreter() { + // python's arity wording has rules a reimplementation keeps getting one short of: + // `and` from two names up, a comma *before* that `and` from three up, a range + // rather than a count once a parameter has a default, and a near miss offered to a + // caller who spelled a keyword almost right. the runtime does not reproduce them — + // it builds a function of the same shape and lets the interpreter refuse the call + // + // the three-name cases are what prove that happened. the wording the runtime keeps + // for itself is deliberately one comma short of python's, so a leg that answered + // from it rather than from the shape would differ here rather than pass + agree_python( + "arityword", + "\ +class Bare: + pass + + +class Three: + def __init__(self, a: int, b: int, c: int) -> None: + self.a = a + + +class Spread: + def __init__(self, a: int, b: int = 1, c: int = 2) -> None: + self.a = a + + +class Box: + def __init__(self) -> None: + self.n = 0 + + def take(self, x: int, y: int, z: int) -> int: + return x + y + z + + +def one(a: int) -> int: + return a + + +def two(a: int, b: int) -> int: + return a + + +def four(a: int, b: int, c: int, d: int) -> int: + return a + + +def named(a: int, *, b: int, c: int, d: int) -> int: + return a + + +def rest(a: int, b: int, *more: int) -> int: + return a +", + &[ + // one name, then two, then three: the separator changes at each step + "str(_capture(m.one))", + "str(_capture(m.two))", + "str(_capture(m.four))", + "str(_capture(m.named, 1))", + "str(_capture(m.rest))", + // a receiver is counted in the arity sentence and not in this one + "str(_capture(lambda: m.Box().take()))", + "str(_capture(lambda: m.Box().take(1, 2, 3, 4)))", + "str(_capture(m.Three))", + "str(_capture(lambda: m.Three(1)))", + // a default turns the count into a range + "str(_capture(lambda: m.Spread(1, 2, 3, 4)))", + // `object.__init__` refuses anything at all, and does not say which kind + "str(_capture(lambda: m.Bare(1)))", + "str(_capture_kw(m.Bare, (), {'q': 1}))", + // a keyword that nearly names a parameter is offered the parameter + "str(_capture_kw(m.two, (1, 2), {'aa': 1}))", + "str(_capture_kw(m.Three, (1, 2, 3), {'aa': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(1, 2, 3, **k), (), {'xx': 1}))", + // and one that nearly names the *synthetic* receiver must not be offered it + "str(_capture_kw(m.two, (1, 2), {'_by_self': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(1, 2, 3, **k), (), {'_by_rest': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(**k), (), {'_by_self': 1}))", + ], + ); +} + #[test] fn a_with_inside_a_generator_agrees() { // the manager has to survive every suspension the body makes, so it lives in a @@ -10390,14 +12929,17 @@ fn a_nested_function_may_be_decorated_or_generic() { return; } // a decorator on a nested function wraps the closure where the `def` stands, - // outermost last — the same order the statement itself applies them. the name - // is resolved the way the module-level ones are, as `LOAD_GLOBAL` would. + // outermost last — the same order the statement itself applies them. the whole + // *expression* is evaluated there, in this frame, which is what lets a call, a + // dotted name and a name the frame itself binds all be taken: `@wraps(fn)` reads + // `fn` out of a register, and `@held` reads the parameter holding the decorator. // // a type parameter is erased here as anywhere else, so a generic nested // function needed nothing beyond dropping the decline agree_python( "nesteddec", "\ +import functools from typing import Callable @@ -10446,6 +12988,38 @@ def counted(n: int) -> list[int]: out.append(each(0)) return out + + +def by_call(n: int) -> Callable[[int], int]: + @scaled(3) + def inner(x: int) -> int: + return x + n + + return inner + + +def by_dotted(n: int) -> Callable[[int], int]: + @functools.cache + def inner(x: int) -> int: + return x + n + + return inner + + +def by_held(n: int, held: Callable[[Callable[[int], int]], Callable[[int], int]]) -> Callable[[int], int]: + @held + def inner(x: int) -> int: + return x + n + + return inner + + +def scaled(k: int) -> Callable[[Callable[[int], int]], Callable[[int], int]]: + def outer(fn: Callable[[int], int]) -> Callable[[int], int]: + def wrapper(x: int) -> int: + return fn(x) * k + return wrapper + return outer ", &[ "m.offset(10)(1)", @@ -10457,6 +13031,13 @@ def counted(n: int) -> list[int]: // each iteration decorates its own closure "m.counted(4)", "m.counted(0)", + // a decorator that is a call, evaluated where the `def` stands + "m.by_call(10)(1)", + "m.by_call(0)(5)", + // a dotted name, read off a module this one imported + "m.by_dotted(10)(1)", + // and one the frame is *holding*, which a global lookup would have missed + "m.by_held(10, m.twice)(1)", ], ); } @@ -11091,7 +13672,7 @@ fn the_arithmetic_dunders_reach_both_directions() { // whether this is `__add__` or `__radd__`. one direction the class does not // define answers `NotImplemented` agree_python( - "arith", + "arithdunders", "\ class Vec: def __init__(self, x: int, y: int) -> None: @@ -11310,7 +13891,7 @@ fn the_container_dunders_fill_their_slots() { // `__setitem__` share the mapping sub-table with `__len__`, and `__contains__` // needs a sequence table of its own agree_python( - "containers", + "containerdunders", "\ class Grid: def __init__(self, items: list[int]) -> None: @@ -12942,30 +15523,226 @@ data class Plain: def doubled(self) -> int: return self.n * 2 -def use(n: int) -> str: - p = Point(n, n + 1) - return str(p.total()) + str(Point.tag) +def use(n: int) -> str: + p = Point(n, n + 1) + return str(p.total()) + str(Point.tag) + +def pairs(s: str) -> str: + q = Pair(s, s) + return q.a + q.b + str(Pair.tag) + str(Pair.made) +", + &[ + "m.use(1)", + "m.pairs('z')", + "m.Point(1, 2).total()", + "m.Point(1, 2).x", + "m.Point.tag", + // both decorators ran, innermost first + "(m.Pair.tag, m.Pair.made)", + // an undecorated class is untouched and keeps its direct method call + "m.Plain(3).doubled()", + // a decorated one is mutable, which is the whole reason it is a heap + // type — an undecorated one stays static, and rejecting `setattr` there + // is a difference from the interpreted class that predates this + "[type(e).__name__ for e in [_capture(setattr, m.Point, 'extra', 1)]]", + "m.Point.extra if hasattr(m.Point, 'extra') else None", + ], + ); +} + +#[test] +fn a_class_decorator_is_handed_the_body_the_class_statement_wrote() { + // a decorator that only *reads* its class still reads the whole of it: the names the + // body bound, the values it gave them and the annotations it wrote. the annotations + // are the ones an emitted type gets last — they are carried over from the twin, and + // that carrying used to happen *after* every decorator had run, so each one was + // handed a class whose body had not arrived yet + agree_python( + "decoratorreads", + "\ +SEEN = {} + + +def inspecting(cls: type) -> type: + SEEN[cls.__name__] = (cls.__annotations__, 'helper' in cls.__dict__, cls.limit) + return cls + + +@inspecting +class Widget: + tag: str + size: int + limit = 7 + + def helper(self) -> int: + return 4 +", + &[ + "sorted(m.SEEN['Widget'][0].items(), key=str)", + "m.SEEN['Widget'][1]", + "m.SEEN['Widget'][2]", + "m.Widget.limit", + "m.Widget().helper()", + ], + ); +} + +#[test] +fn a_dataclass_decorator_builds_a_constructor_that_runs() { + // `@dataclass` does not read a class so much as *generate* from it — an `__init__` + // taking one argument per annotation and assigning one attribute each — so it is the + // strictest question that can be asked about how faithful an emitted class is. it + // found two answers wrong at once: an empty `__annotations__` made it write + // `__init__(self)`, and once that was fixed the assignments had nowhere to land, + // because an emitted instance is its layout and this class has no fields at all. + // + // `y` carries a default, which is a class-level value the decorator reads and then + // rewrites: `_process_class` leaves the bare `5` where the `field(default=5)` stood. + // so the value the emitted type carries has to be the one the body wrote rather than + // the one left behind, and `_capture(m.Point)` is where that shows — a `Point` built + // with no arguments at all + agree_python( + "dataclassdecorator", + "\ +from dataclasses import dataclass, field, fields, replace + + +@dataclass +class Point: + x: int + y: int = field(default=5, repr=False) + + def total(self) -> int: + return self.x + self.y + + +def make(n: int) -> str: + return repr(Point(n, 5)) +", + &[ + "m.make(3)", + "m.Point(1, 2)", + "m.Point(1)", + "m.Point(1, 2) == m.Point(1, 2)", + "m.Point(1, 2) == m.Point(1, 3)", + "m.Point(1, 2).total()", + "[(f.name, f.type, f.repr) for f in m.fields(m.Point)]", + "sorted(m.Point(1, 2).__dict__.items())", + "m.Point.__match_args__", + "m.replace(m.Point(1, 2), y=9)", + "[(type(e).__name__, str(e)) for e in [_capture(m.Point)]]", + ], + ); +} + +#[test] +fn a_decorated_class_is_the_compiled_one_and_its_dict_is_collectable() { + // the differential legs agree whichever class answered, so this is where the + // compiled one is pinned down. it is not a formality: the managed dict this class + // now carries moves `tp_dictoffset` off its base's, which `By_OffsetsHoldUp` read as + // the wrong kind of inheritance — the type was quietly dropped for its interpreted + // twin, and every compiled function went on reading that twin's instances as its own + // struct. `m.on_final(m.Fixed(2, 'x'))` answered a pointer + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_decoratedlive"); + let _ = std::fs::remove_dir_all(&dir); + // `from __future__ import annotations` is what keeps `through`'s parameter from + // being a *read* of `Derived`: without it the module evaluates the annotation where + // the `def` stands, which is inside the window where `Derived` still holds the + // interpreted definition, and the class declines rather than move its decorator + let source = "\ +from __future__ import annotations + +from dataclasses import dataclass + + +def tagged(cls: type) -> type: + cls.tag = 'seen' + return cls + + +@dataclass +class Point: + x: int + + def doubled(self) -> int: + return self.x * 2 + + +@tagged +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +class Plain: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +@tagged +class Derived(Plain): + def tripled(self) -> int: + return self.n * 3 -def pairs(s: str) -> str: - q = Pair(s, s) - return q.a + q.b + str(Pair.tag) + str(Pair.made) -", - &[ - "m.use(1)", - "m.pairs('z')", - "m.Point(1, 2).total()", - "m.Point(1, 2).x", - "m.Point.tag", - // both decorators ran, innermost first - "(m.Pair.tag, m.Pair.made)", - // an undecorated class is untouched and keeps its direct method call - "m.Plain(3).doubled()", - // a decorated one is mutable, which is the whole reason it is a heap - // type — an undecorated one stays static, and rejecting `setattr` there - // is a difference from the interpreted class that predates this - "[type(e).__name__ for e in [_capture(setattr, m.Point, 'extra', 1)]]", - "m.Point.extra if hasattr(m.Point, 'extra') else None", - ], + +def through(d: Derived) -> int: + return d.tripled() +"; + let built = match build_source( + source, + "by_diff_decoratedlive", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import gc\n\ + import by_diff_decoratedlive as m\n\ + print(type(m.Point.doubled).__name__, type(m.Held.read).__name__,\n\ + \x20 type(m.Plain.read).__name__, type(m.Derived.tripled).__name__)\n\ + print(m.Point(3).doubled(), m.Held(4).read(), m.Plain(5).read())\n\ + print(m.Point(3).__dict__, m.Held.tag)\n\ + print(m.through(m.Derived(2)), m.Derived(2).read())\n\ + gc.collect()\n\ + base = len(gc.get_objects())\n\ + for _ in range(200):\n\ + \x20 h = m.Held(1)\n\ + \x20 h.itself = h\n\ + del h\n\ + gc.collect()\n\ + print('collected' if len(gc.get_objects()) <= base + 10 else 'leaked')\n\ + print(gc.is_tracked(m.Held(1)), gc.is_tracked(m.Plain(1)))\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor method_descriptor method_descriptor\n\ + 6 4 5\n\ + {'x': 3} seen\n\ + 6 2\n\ + collected\n\ + True False" ); } @@ -14180,3 +16957,311 @@ def carried(n: int) -> object: ], ); } + +#[test] +fn a_module_level_name_bound_to_a_class_follows_the_class_it_named() { + // the whole module body runs against the interpreted definitions, so a name it binds + // to a class holds that definition — and the compiled type only ever replaces the one + // name the `class` statement wrote. what that left was two classes of the same name in + // the same module: `Kind()` built an object `isinstance(obj, C)` denied, and the + // compiled `hi` refused it outright. + // + // a name that *is* a twin is the one shape that moves soundly, so it is moved onto + // whatever stands under the class's own name. the annotation on it is beside the + // point: `Kind: type[C] = C` and `Bare = C` are the same binding. + // + // `method_descriptor` is what says the compiled type answered at all — a class that + // fell back to its interpreted definition has one class and could not show this + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_classalias"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class C: + def hi(self) -> int: + return 7 + + +Kind: type[C] = C +Bare = C +Rebound = Bare +"; + let built = match build_source( + source, + "by_diff_classalias", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_classalias as m\n\ + print(m.Kind is m.C, m.Bare is m.C, m.Rebound is m.C)\n\ + print(isinstance(m.Kind(), m.C), isinstance(m.C(), m.Bare))\n\ + print(m.C.hi(m.Kind()), m.Kind().hi())\n\ + print(type(m.C.hi).__name__)\n", + ); + assert_eq!( + out, + "True True True\n\ + True True\n\ + 7 7\n\ + method_descriptor" + ); +} + +#[test] +fn a_class_constant_naming_another_class_is_the_type_that_replaced_it() { + // a class-level constant is taken off the interpreted definition, so `attr = C` in a + // body hands over the *twin* — and copying that verbatim gave the compiled type an + // attribute naming a class nothing else in the module could reach. it goes through the + // same substitution every carried attribute does. + // + // `held` is the boundary in the other direction: a constant that only *reaches* a twin + // cannot be substituted, because the tuple is the object the body built and its + // identity is not the twin's. it is pinned here as still *present*, because dropping + // those instead was built and backed out on the measurement — 65 attributes lost over + // the stdlib, `ipaddress`'s network constants among them. what the reach holds is a + // defect of its own and is left exactly as the interpreted definition had it + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_classconst"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class C: + def hi(self) -> int: + return 7 + + +class Holder: + attr = C + held = (C,) + plain = 3 + + def tag(self) -> str: + return \"holder\" +"; + let built = match build_source( + source, + "by_diff_classconst", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_classconst as m\n\ + print(m.Holder.attr is m.C, m.Holder.plain)\n\ + print(m.C.hi(m.Holder.attr()))\n\ + print(hasattr(m.Holder, 'held'), len(m.Holder.held))\n\ + print(type(m.C.hi).__name__, type(m.Holder.tag).__name__)\n", + ); + assert_eq!( + out, + "True 3\n\ + 7\n\ + True 1\n\ + method_descriptor method_descriptor" + ); +} + +#[test] +fn a_base_written_as_an_alias_reaches_the_class_it_was_bound_to() { + // `Alias` and `Root` are one class, so the two spellings have to build one class. + // taking the alias for a name out of this module built `Over` on the interpreted + // definition instead — the emitted type goes into the namespace under `Root`, and an + // alias is carried over to it only once every class has been built — so + // `isinstance(Over(), Root)` answered `False` where python answers `True`, while + // `Over.__mro__` still said `Root`. a wrong answer, and one no sweep reaches: the + // class builds, imports, constructs and subclasses with only its contents lying + agree_python( + "aliasbase", + "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +", + &[ + "isinstance(m.Over(), m.Root)", + "m.Over.__mro__[1] is m.Root", + "[c.__name__ for c in m.Over.__mro__]", + "m.Over().root()", + "m.Over().side()", + "issubclass(m.Over, m.Root)", + ], + ); +} + +#[test] +fn an_alias_reaches_the_compiled_type_rather_than_the_twin() { + // the same source again, because `agree` cannot say which build answered: the + // interpreted definition answers every one of those calls identically. what is + // wanted is that the *compiled* type stands under the name, and `method_descriptor` + // against `function` is what says so + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_aliasbase_type"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + let built = match build_source( + source, + "by_diff_aliasbase_type", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_aliasbase_type as m\n\ + print(type(m.Root.root).__name__, type(m.Over.side).__name__)\n\ + print(m.Alias is m.Root, m.Over.__mro__[1] is m.Root)\n\ + print(isinstance(m.Over(), m.Root))\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + True True\n\ + True" + ); +} + +#[test] +fn an_alias_does_not_carry_a_base_this_module_lays_out_past_the_gate() { + // `class OnLaid(Laid, codecs.Codec)` is refused because the layout would have to be + // inherited from outside and laid out here at once. written through an alias it went + // straight past — the gate asks `layouts` about the *name* — and compiled the one + // shape it exists to refuse. so the refusal has to survive the spelling, and `Laid` + // has to go interpreted with it or `isinstance` disagrees for the same reason again + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_aliaslaid"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import codecs + + +class Laid: + def __init__(self, n: int) -> None: + self.n = n + + def held(self) -> int: + return self.n + + +Alias = Laid + + +class OnLaid(Alias, codecs.Codec): + def side(self) -> int: + return 2 +"; + let built = match build_source( + source, + "by_diff_aliaslaid", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "OnLaid", + "a base this module lays out cannot stand beside one it does not" + ), + ( + "Laid", + "`OnLaid` declined, so it extends the interpreted definition rather than this type" + ) + ] + ); + let out = run( + &python, + &dir, + "import by_diff_aliaslaid as m\n\ + print(m.Alias is m.Laid, m.OnLaid.__mro__[1] is m.Laid)\n\ + print(isinstance(m.OnLaid(1), m.Laid), m.OnLaid(1).held(), m.OnLaid(1).side())\n\ + print(type(m.Laid.held).__name__, type(m.OnLaid.side).__name__)\n", + ); + assert_eq!( + out, + "True True\n\ + True 1 2\n\ + function function" + ); +} diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 48b7056309..18ccc3e96a 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -18,7 +18,7 @@ use std::process::Command; use by_build::{Options, Toolchain, build_module, build_source}; use by_ir::builder::FunctionBuilder; -use by_ir::function::{CallConvention, ModuleIr}; +use by_ir::function::{CallConvention, ModuleIr, ModuleName}; use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; use by_ir::rtype::RType; @@ -114,7 +114,7 @@ fn arith_module() -> ModuleIr { builder.terminate(Terminator::Return(Value::Register(result))); ModuleIr { - name: "by_e2e_arith".to_string(), + name: by_ir::ModuleName::new("by_e2e_arith"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -178,7 +178,7 @@ fn fib_module() -> ModuleIr { builder.terminate(Terminator::Return(Value::Register(a))); ModuleIr { - name: "by_e2e_fib".to_string(), + name: by_ir::ModuleName::new("by_e2e_fib"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -314,7 +314,7 @@ fn division_floors_like_python_and_raises_on_zero() { }); builder.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "by_e2e_div".to_string(), + name: by_ir::ModuleName::new("by_e2e_div"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -370,7 +370,7 @@ fn floats_are_unboxed_and_exclude_int() { }); builder.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "by_e2e_float".to_string(), + name: by_ir::ModuleName::new("by_e2e_float"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -441,7 +441,7 @@ fn calls_between_compiled_functions_stay_native() { quad.terminate(Terminator::Return(Value::Register(twice))); let module = ModuleIr { - name: "by_e2e_call".to_string(), + name: by_ir::ModuleName::new("by_e2e_call"), functions: vec![double.finish(), quad.finish()], declined: Vec::new(), classes: Vec::new(), @@ -701,7 +701,7 @@ fn a_stdlib_module_compiles_without_a_hard_failure(name: &str) { }; match build_source( &source, - &module, + module.as_str(), &toolchain, &dir.join(&module), &Options::default(), @@ -1122,3 +1122,97 @@ fn the_emitted_c_names_no_pointer_type_it_does_not_mean() { "the C compiler rejected the generated code:\n{stderr}" ); } + +/// the source of a package member, whose answers say which member is speaking +/// +/// `tag` is the same name in both `dup` modules, which is the whole point: a flat +/// output directory called both files `dup`, so the second silently replaced the +/// first and neither was importable under the name it was compiled as +fn member_source(tag: i32) -> String { + format!( + "class Member:\n def __init__(self) -> None:\n self.tag: int = {tag}\n\n\ndef tag() -> int:\n return {tag}\n" + ) +} + +/// the artefacts of a package build have to be laid out as the package, because +/// that is the only shape cpython's finder will import them back under: it looks +/// for `pkg/sub/dup` for a member and `pkg/sub/__init__` for the +/// package itself, and never for a flat file named after the last component +#[test] +fn a_package_is_built_as_a_tree_and_imports_under_its_dotted_names() { + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 12)) { + return; + } + let dir = std::env::temp_dir().join("by_e2e_package_tree"); + let _ = std::fs::remove_dir_all(&dir); + + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + // both packages and both members go into one output directory, the way + // `by compile -o` builds a whole project + let members = [ + (ModuleName::package("by_e2e_pkg"), 1), + (ModuleName::package("by_e2e_pkg.sub"), 2), + (ModuleName::new("by_e2e_pkg.dup"), 3), + (ModuleName::new("by_e2e_pkg.sub.dup"), 4), + ]; + for (name, tag) in &members { + let Ok(built) = build_source( + &member_source(*tag), + name.clone(), + &toolchain, + &dir, + &options, + ) else { + eprintln!("skipping: no working C toolchain"); + return; + }; + assert!( + built.artifact.extension.exists(), + "{} was written to {}", + name.dotted(), + built.artifact.extension.display() + ); + } + + // four distinct artefacts: the two `dup` members used to be one file + assert_eq!( + members + .iter() + .map(|(name, _)| toolchain.extension_path(name)) + .collect::>() + .len(), + 4 + ); + + let printed = script( + &python, + &dir, + "import sys\n\ + import by_e2e_pkg.sub.dup\n\ + import by_e2e_pkg.dup\n\ + for name in ('by_e2e_pkg', 'by_e2e_pkg.sub', 'by_e2e_pkg.dup', 'by_e2e_pkg.sub.dup'):\n\ + \x20 m = sys.modules[name]\n\ + \x20 print(name, m.__name__, m.tag(), m.Member.__module__, m.__file__)\n", + ); + let lines: Vec<&str> = printed.lines().collect(); + assert_eq!(lines.len(), 4, "{printed}"); + for (line, (name, tag)) in lines.iter().zip(&members) { + let fields: Vec<&str> = line.split_whitespace().collect(); + // the key it answered to in `sys.modules`, the name the module reports, + // and the module a class written in it belongs to all have to be the one + // it was compiled as + assert_eq!(fields[0], name.dotted(), "{printed}"); + assert_eq!(fields[1], name.dotted(), "{printed}"); + assert_eq!(fields[2], tag.to_string(), "{printed}"); + assert_eq!(fields[3], name.dotted(), "{printed}"); + // and it answered from the extension, not from some interpreted source + // that happened to be lying beside it + assert!(fields[4].ends_with(&toolchain.ext_suffix), "{printed}"); + } +} diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index eb8ff0909e..8427a3a1de 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -26,7 +26,7 @@ use std::collections::BTreeSet; use std::fmt::Write; use by_ir::function::{ - ClassBase, ClassIr, Function, KeywordValue, ModuleIr, RegisterDecl, Surface, + Binding, ClassBase, ClassIr, Function, KeywordValue, ModuleIr, RegisterDecl, Surface, }; use by_ir::ops::{BinOp, BlockId, CmpOp, Mutation, Op, RegisterId, Terminator, UnaryOp, Value}; use by_ir::rtype::{Primitive, RType, tuple_mangle}; @@ -125,7 +125,7 @@ pub fn emit_module(module: &ModuleIr) -> String { let _ = writeln!( out, "static PyObject *{} = NULL;", - function.interpreted_symbol(&module.name) + function.interpreted_symbol(module.name.dotted()) ); } } @@ -181,7 +181,7 @@ pub fn emit_module(module: &ModuleIr) -> String { // so its name is a pointer rather than a static struct. only a generator's state // and a closure's environment stay static let heap = heap_type(module, class); - let type_name = class.type_name(&module.name); + let type_name = class.type_name(module.name.dotted()); let _ = writeln!( out, "{} {type_name};", @@ -218,7 +218,7 @@ pub fn emit_module(module: &ModuleIr) -> String { let _ = writeln!( out, "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames);", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } } @@ -233,8 +233,13 @@ pub fn emit_module(module: &ModuleIr) -> String { out.push('\n'); // a method's wrapper takes the receiver from the `self` slot rather // than from the argument vector, which is how `METH_FASTCALL` on a - // type presents it - out.push_str(&emit_wrapper(module, method, true)); + // type presents it. a `staticmethod` has nothing there and binds every + // parameter it declares, exactly as a module-level function does + out.push_str(&emit_wrapper( + module, + method, + method.binding.takes_slot_zero_from_self(), + )); out.push('\n'); } } @@ -332,6 +337,56 @@ fn emit_tuple_struct(module: &ModuleIr, items: &[RType]) -> String { out } +/// whether the collector can follow this field, which it can where the field is held as +/// a plain `PyObject *` +/// +/// a tagged `int` is either not an object at all or a `PyLong`, and neither can be part +/// of a cycle; an unboxed buffer holds no references either +fn collectable(field: &by_ir::function::FieldDecl) -> bool { + matches!( + field.ty, + RType::Instance { .. } + | RType::Primitive(Primitive::Object | Primitive::Str | Primitive::List) + ) +} + +/// `tp_traverse` and `tp_clear` for a class that owns its layout and keeps an instance +/// dict beside it +/// +/// the fields are this class's whole struct — a subclass's begins with its base's and is +/// cloned into it, so there is no base to chain to and nothing of the base's is missed. +/// the type is visited because an instance of a heap type holds a reference to it, and +/// the dict is visited because that is the whole reason the type is collected at all +fn emit_collected_instance(module: &ModuleIr, class: &ClassIr) -> String { + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); + let mut visits = String::new(); + let mut clears = String::new(); + for field in class.fields.iter().filter(|field| collectable(field)) { + let _ = writeln!( + out_slot(&mut visits), + " Py_VISIT(self->{});", + field.member() + ); + let _ = writeln!( + out_slot(&mut clears), + " Py_CLEAR(self->{});", + field.member() + ); + } + format!( + "static int {type_name}_traverse({struct_name} *self, visitproc visit, void *arg) {{\n\ + \x20 Py_VISIT(Py_TYPE(self));\n\ + {visits}\ + \x20 By_VisitManagedDict((PyObject *)self, visit, arg);\n\ + \x20 return 0;\n}}\n\n\ + static int {type_name}_clear({struct_name} *self) {{\n\ + {clears}\ + \x20 By_ClearManagedDict((PyObject *)self);\n\ + \x20 return 0;\n}}\n\n" + ) +} + /// `tp_dealloc`, `tp_traverse` and `tp_clear` for a class whose fields sit past a /// base's instance /// @@ -341,8 +396,8 @@ fn emit_tuple_struct(module: &ModuleIr, items: &[RType]) -> String { /// optional either: a base like `Exception` is a GC type, so ours is, and a field the /// collector cannot see holds its cycle alive forever fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); // the *declaring* type, not `Py_TYPE(self)`: a python subclass of this class is a // different type whose data area is somewhere else again, and the base to chain to // is this class's base rather than that subclass's @@ -350,19 +405,9 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { let fields = format!("({struct_name} *)By_TypeData(self, {declared})"); let mut out = String::new(); - // only a field the collector can follow, which is one held as a plain `PyObject *`. - // a tagged `int` is either not an object at all or a `PyLong`, and neither can be - // part of a cycle; an unboxed buffer holds no references either - let collected = |field: &&by_ir::function::FieldDecl| { - matches!( - field.ty, - RType::Instance { .. } - | RType::Primitive(Primitive::Object | Primitive::Str | Primitive::List) - ) - }; let mut visits = String::new(); let mut clears = String::new(); - for field in class.fields.iter().filter(collected) { + for field in class.fields.iter().filter(|field| collectable(field)) { let _ = writeln!( out_slot(&mut visits), " Py_VISIT(by_f->{});", @@ -375,9 +420,13 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { ); } - // `subtype_traverse` visits the type only when the base is not itself a heap type, - // which is exactly the case a subclass of this class is *not* in — so this one has - // to, or the type is never collected + // an instance of a heap type counts as a reference to that type, and the collector + // has to see it or a cycle through the type is never broken. exactly one traverse in + // the chain reports it, which is `subtype_traverse`'s own rule: the one whose base + // does not itself carry the link. a base out of this module is not a heap type — the + // construction refuses one that is — so this traverse is that one; a base *this* + // module appends to is, and its traverse has already reported it. counting it twice + // would tell the collector the instance holds two references where it holds one let _ = write!( out, "static int {type_name}_traverse(PyObject *self, visitproc visit, void *arg) {{\n\ @@ -388,7 +437,7 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { \x20 int by_r = by_base->tp_traverse(self, visit, arg);\n\ \x20 if (by_r) return by_r;\n\ \x20 }}\n\ - \x20 Py_VISIT(Py_TYPE(self));\n\ + \x20 if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_VISIT(Py_TYPE(self));\n\ \x20 return 0;\n}}\n\n\ static int {type_name}_clear(PyObject *self) {{\n\ \x20 {struct_name} *by_f = {fields};\n\ @@ -410,16 +459,21 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out_slot(&mut releases), " {release}"); } } + // and dropped by exactly one rung, the same one the traverse reports it from: a base + // that is itself a heap type has a deallocator of its own that drops it, and two + // drops for the one reference free the type underneath everything still using it let _ = write!( out, "static void {type_name}_dealloc(PyObject *self) {{\n\ \x20 PyTypeObject *by_type = Py_TYPE(self);\n\ + \x20 PyTypeObject *by_base = {declared}->tp_base;\n\ \x20 if (PyType_HasFeature(by_type, Py_TPFLAGS_HAVE_GC)) PyObject_GC_UnTrack(self);\n\ \x20 {{ {struct_name} *by_f = {fields};\n\ {releases}\ \x20 }}\n\ - \x20 {declared}->tp_base->tp_dealloc(self);\n\ - \x20 if (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE) Py_DECREF(by_type);\n}}\n\n" + \x20 by_base->tp_dealloc(self);\n\ + \x20 if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)\n\ + \x20 && (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_DECREF(by_type);\n}}\n\n" ); out } @@ -436,7 +490,7 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { }; let mut out = format!( "typedef struct {} {{\n{header}", - class.struct_name(&module.name) + class.struct_name(module.name.dotted()) ); for field in &class.fields { let _ = writeln!(out, " {} {};", ctype(module, &field.ty), field.member()); @@ -446,7 +500,7 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out, " char {};", field.presence()); } } - let _ = writeln!(out, "}} {};", class.struct_name(&module.name)); + let _ = writeln!(out, "}} {};", class.struct_name(module.name.dotted())); out } @@ -454,18 +508,27 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { /// releases them. every field is written by `__init__`, which is what makes them /// *always defined* — no bitfield and no per-read check fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); let mut out = String::new(); + let keeps_a_dict = instance_dict(module, class); if external_storage(module, class) { out.push_str(&emit_appended_storage(module, class)); } else { + if keeps_a_dict { + out.push_str(&emit_collected_instance(module, class)); + } // dealloc releases each refcounted field, then the object let _ = writeln!( out, "static void {type_name}_dealloc({struct_name} *self) {{" ); + // a collected instance is on the collector's list until it says otherwise, and + // a list holding a half-freed object is what the next collection walks + if keeps_a_dict { + out.push_str(" PyObject_GC_UnTrack(self);\n"); + } // a finalizer does not run itself: `subtype_dealloc` calls it, and a type that // writes its own dealloc has to do the same or the cleanups never happen. a // negative answer means the finalizer resurrected the object, and freeing it @@ -479,6 +542,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { " if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) return;\n", ); } + if keeps_a_dict { + out.push_str(" By_ClearManagedDict((PyObject *)self);\n"); + } for field in &class.fields { if let Some(release) = dec_ref(&field.ty, &format!("self->{}", field.member())) { let _ = writeln!(out, " {release}"); @@ -712,7 +778,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { out, " if ({}) return By_InitInterpreted({}, {fname}, selfobj, args, kwds);", tests.join(" || "), - init.interpreted_symbol(&module.name) + init.interpreted_symbol(module.name.dotted()) ); } let params = init.params().get(1..).unwrap_or_default(); @@ -779,7 +845,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { out, " {{ {} by_result = {}(({receiver})selfobj{arguments});", ctype(module, &init.ret), - init.native_symbol(&module.name) + init.native_symbol(module.name.dotted()) ); if init.convention.can_fail() { let _ = writeln!( @@ -850,8 +916,8 @@ fn mark_present(field: &by_ir::function::FieldDecl) -> String { /// the getters, setters, slot table and type spec python sees fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); let mut out = String::new(); // getters and setters, so python sees ordinary attributes @@ -921,6 +987,16 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { field.name, field.name ); } + // a class keeping an instance dict answers `__dict__` with it — but only where the + // dict is the whole of an instance's state. a class with fields of its own keeps + // them in its layout, and a mapping that named none of them would be an *empty* + // answer where the interpreted class gives a full one: quiet, and wrong. the + // refusal such a class already gives is at least loud + if instance_dict(module, class) && class.fields.is_empty() { + out.push_str( + " {\"__dict__\", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL},\n", + ); + } out.push_str(" {NULL, NULL, NULL, NULL, NULL}\n};\n\n"); // the method table, using each method's python wrapper @@ -935,7 +1011,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .methods .iter() .find(|method| method.name == resume.method) - .map(|method| method.native_symbol(&module.name)) + .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); let _ = writeln!( out, @@ -1052,11 +1128,20 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ); } for method in &class.methods { + // `METH_STATIC` and `METH_CLASS` are masked off before the calling convention + // is read, so either combines with the fastcall the wrapper is written for. + // what they change is the descriptor the type publishes — a `staticmethod` or a + // `classmethod_descriptor` rather than a plain `method_descriptor` let _ = writeln!( out, - " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS, NULL}},", + " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS{}, NULL}},", method.name, - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()), + method + .binding + .method_flag() + .map(|flag| format!(" | {flag}")) + .unwrap_or_default() ); } out.push_str(" {NULL, NULL, 0, NULL}\n};\n\n"); @@ -1070,7 +1155,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .methods .iter() .find(|method| method.name == resume.method) - .map(|method| method.native_symbol(&module.name)) + .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); let _ = writeln!( out, @@ -1086,7 +1171,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .find(|field| field.name == "$kind") .map(by_ir::FieldDecl::member) .unwrap_or_default(); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let dotted = module.name.dotted(); if resume.surface == Surface::AsyncGenerator { // `__anext__` hands back an awaitable rather than an item, because the // body may `await` before it reaches its next `yield`. one `resume` @@ -1145,7 +1230,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 .am_await = {type_name}_asend_await,\n}};\n\ static PyTypeObject {type_name}_asend_type = {{\n\ \x20 PyVarObject_HEAD_INIT(NULL, 0)\n\ - \x20 .tp_name = \"{last}.{}.ascend\",\n\ + \x20 .tp_name = \"{dotted}.{}.ascend\",\n\ \x20 .tp_basicsize = sizeof({type_name}_asend),\n\ \x20 .tp_dealloc = (destructor){type_name}_asend_dealloc,\n\ \x20 .tp_flags = Py_TPFLAGS_DEFAULT,\n\ @@ -1233,13 +1318,13 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out.push_str(&emit_dunder_adapters(module, class, &type_name)); let dunders = dunder_initializers(class, &type_name); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let dotted = module.name.dotted(); // a static struct is what a class no name reaches gets: a generator's state or a // closure's environment, neither of which anything can ask about if !heap_type(module, class) { let _ = write!( out, - "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{last}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n .tp_flags = Py_TPFLAGS_DEFAULT,\n{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ + "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n .tp_flags = Py_TPFLAGS_DEFAULT,\n{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ ", class.name ); @@ -1269,6 +1354,11 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // the base happens to be collected let collected = if external_storage(module, class) { " | Py_TPFLAGS_HAVE_GC" + } else if instance_dict(module, class) { + // the dict holds whatever a decorator's generated code put in it, so the + // collector has to be able to walk it — and a managed one is only allowed on a + // type it can walk + " | Py_TPFLAGS_HAVE_GC | BY_MANAGED_DICT_FLAG" } else { "" }; @@ -1317,9 +1407,21 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } else { format!("{init}\x20 {{Py_tp_new, (void *)PyType_GenericNew}},\n") }; + // a collected type has to hand the collector both halves, or an instance in a + // cycle is never reached at all + let walked = if instance_dict(module, class) { + format!( + "\x20 {{Py_tp_traverse, (void *){type_name}_traverse}},\n\ + \x20 {{Py_tp_clear, (void *){type_name}_clear}},\n" + ) + } else { + String::new() + }; ( format!("sizeof({struct_name})"), - format!("\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n{construction}"), + format!( + "\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n{walked}{construction}" + ), ) }; let _ = write!( @@ -1330,7 +1432,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {{Py_tp_getset, (void *){type_name}_getset}},\n\ {slots}\x20 {{0, NULL}},\n}};\n\ static PyType_Spec {type_name}_spec = {{\n\ - \x20 \"{last}.{}\",\n\ + \x20 \"{dotted}.{}\",\n\ \x20 {basicsize},\n\ \x20 0,\n\ \x20 Py_TPFLAGS_DEFAULT{basetype}{collected},\n\ @@ -1342,6 +1444,53 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out } +/// whether this class or an in-module base of it carries a class decorator +/// +/// the base chain, because an instance discipline is not a per-class answer: a subclass +/// allocates and frees instances of a shape its base decided +fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { + let mut current = class; + // bounded by the class count, for the reason `inherits_layout` gives + for _ in 0..=module.classes.len() { + if !current.decorators.is_empty() { + return true; + } + match current + .base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|name| class_named(module, name)) + { + Some(next) => current = next, + None => return false, + } + } + false +} + +/// whether an emitted instance keeps a `__dict__` of its own +/// +/// a class decorator is arbitrary python handed the class, and what it hands back is +/// often code it *generated* from what it read — `@dataclass` writes an `__init__` that +/// assigns one attribute per annotation. that code is ordinary python and assumes an +/// ordinary instance, so on an emitted one, whose whole state is its layout, every +/// assignment falls off: `E(3)` raised where the interpreted class answered. the decline +/// that would otherwise be the answer is not open here — a class has no runtime fallback +/// — so the class is given the one thing the generated code needs instead. +/// +/// a managed dict costs the layout nothing: python keeps it in the pre-header, so the +/// struct, its base's prefix and every offset a compiled function reads are untouched. +/// what it costs is collection — a type with a dict of arbitrary values must be one the +/// collector walks — which is why only the classes that need it take it. +/// +/// only a class that owns its layout from `object`. one standing on a base outside the +/// module takes that base's answer about a dict, and a spec claiming one anyway would be +/// claiming room the base never allocated — which is how 24 of the `encodings` modules +/// once segfaulted +fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { + heap_type(module, class) && !inherits_layout(module, class) && decorated_chain(module, class) +} + /// whether this class takes its instance layout from a base outside the module /// /// transitively: an in-module base that itself extends an external one has no layout of @@ -1440,6 +1589,41 @@ fn appends_storage_from_a_spec(module: &ModuleIr, class: &ClassIr) -> bool { && !stands_on_an_emitted_base(module, class) } +/// the class this one appends its storage to, where that is one this module also builds +/// from a spec +/// +/// such a base is the one heap type a spec can be built on: its `tp_dealloc`, +/// `tp_traverse` and `tp_clear` are ones this module emitted, and each of those reads +/// the base to chain to from the type that *declared* it rather than from +/// `Py_TYPE(self)` — so the chain walks down to the outside base and stops, where +/// `subtype_dealloc` would come straight back. see `By_SpecSubclass`. +/// +/// the base has to come first in the module's order, because that is the order module +/// init builds them in and the subclass's spec stands on the finished type. a class +/// statement cannot name a base declared after it, so this only ever rules out a shape +/// the source could not have written. +/// +/// and the subclass has to declare only what it *adds*. the other layout model for an +/// in-module base is the struct extension, where a subclass restates its base's fields +/// so that a pointer to one is a pointer to the other — restating them in an appended +/// region instead would give the pair two copies of each, and the base's methods and the +/// subclass's would write different ones. so a field the base already stores is the +/// signal that this is the other model, and it is not appended over anything +fn appended_over_an_emitted_base<'a>(module: &'a ModuleIr, class: &ClassIr) -> Option<&'a ClassIr> { + let wanted = class.base.as_ref()?.in_module()?; + let base = module + .classes + .iter() + .take_while(|candidate| candidate.name != class.name) + .find(|candidate| candidate.name == wanted)?; + let restates = class.fields.iter().any(|field| { + base.fields + .iter() + .any(|inherited| inherited.name == field.name) + }); + (appends_storage_from_a_spec(module, base) && !restates).then_some(base) +} + /// whether this class can be built the way a `class` statement builds one — by calling /// its metaclass — rather than from a type spec /// @@ -1450,23 +1634,19 @@ fn appends_storage_from_a_spec(module: &ModuleIr, class: &ClassIr) -> bool { /// or one it inherits, since a subclass's struct begins with its base's. everything /// else a spec would have carried — the methods, and through them the type slots — /// goes in the namespace, which is where python puts it too +/// +/// a class-level constant goes in the namespace as well, so it is not a reason to keep a +/// class off this construction. what it *is* is a reason to check afterwards: the +/// namespace is where a metaclass reinterprets what the body wrote, and an `EnumType` +/// handed `STRICT = 'strict'` builds a member the module body's references do not name. +/// `By_ConstantsHeldUp` is that check, and where it fails the interpreted definition +/// stands — the same answer such a class had when this said no to it outright fn metaclass_construction(class: &ClassIr) -> bool { class.fields.is_empty() // a resumable class is a generator's state object: its state *is* its fields, // so this is already false, and nothing in the language can name it as a base && class.resume.is_none() && !decorates_a_method(class) - // a class-level constant is copied onto the finished type for the same reason a - // method decorator is applied there, and it disagrees with the metaclass the same - // way. the copy is sound under `type`, which reads nothing out of the namespace - // and leaves the value the object it already was — but a metaclass that *makes* - // something of what the body wrote never sees it. an `EnumType` handed a - // memberless namespace declares no members, and the constants land in the type's - // dict behind its back: `FlagBoundary.STRICT` answers while `_member_names_` is - // empty. feeding the twin's finished attributes into the namespace instead does - // not rescue it, because the metaclass would build *new* members and every - // reference the module body already took would still name the old ones - && class.constants.is_empty() } /// whether any of this class's methods carries a decorator @@ -1492,11 +1672,11 @@ fn class_named<'a>(module: &'a ModuleIr, name: &str) -> Option<&'a ClassIr> { /// that has always been emitted. for one appending to a base it is a lookup, because /// the offset depends on a base size known only at runtime fn fields_of(module: &ModuleIr, class: &ClassIr, object: &str) -> String { - let struct_name = class.struct_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); if external_storage(module, class) { return format!( "(({struct_name} *)By_TypeData({object}, {}_OBJ))", - class.type_name(&module.name) + class.type_name(module.name.dotted()) ); } format!("({struct_name} *){object}") @@ -1509,7 +1689,7 @@ fn fields_of(module: &ModuleIr, class: &ClassIr, object: &str) -> String { fn bind_self(module: &ModuleIr, class: &ClassIr, object: &str) -> String { format!( "{} *self = {};", - class.struct_name(&module.name), + class.struct_name(module.name.dotted()), fields_of(module, class, object) ) } @@ -1830,7 +2010,7 @@ fn on_our_operand( \x20 PyObject *by_argv[] = {{ {} }};\n\ \x20 return {}({receiver}, by_argv, {}, NULL);\n }}\n", args.join(", "), - method.wrapper_symbol(&module.name), + method.wrapper_symbol(module.name.dotted()), args.len() ) } @@ -1944,7 +2124,7 @@ fn emit_ass_subscript_adapter(module: &ModuleIr, class: &ClassIr, symbol: &str) \x20 if (by_r == NULL) return -1;\n\ \x20 Py_DECREF(by_r);\n\ \x20 return 0;\n", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ), None => format!( " PyErr_SetString(PyExc_AttributeError, {});\n\x20 return -1;\n", @@ -1984,7 +2164,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> }; let call = format!( "{}(self, NULL, 0, NULL)", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); match shape { SlotShape::Unary => { @@ -2010,7 +2190,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> "static PyObject *{symbol}(PyObject *self, PyObject *by_arg) {{\n\ \x20 PyObject *by_argv[] = {{ by_arg }};\n\ \x20 return {}(self, by_argv, 1, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } // emitted above, from both of the methods that fill this slot @@ -2023,7 +2203,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> out, "static PyObject *{symbol}(PyObject *self, PyObject *by_args, PyObject *by_kw) {{\n\ \x20 return By_CallSlot({}, self, by_args, by_kw);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::GetAttrHook => { @@ -2039,7 +2219,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 PyErr_Clear();\n\ \x20 PyObject *by_argv[] = {{ by_name }};\n\ \x20 return {}(self, by_argv, 1, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Finalize => { @@ -2068,7 +2248,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 PyObject *by_argv[] = {{ by_obj ? by_obj : Py_None,\n\ \x20 by_type ? by_type : Py_None }};\n\ \x20 return {}(self, by_argv, 2, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Contains => { @@ -2081,7 +2261,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 int by_v = PyObject_IsTrue(by_r);\n\ \x20 Py_DECREF(by_r);\n\ \x20 return by_v;\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Hash => { @@ -2125,7 +2305,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> let _ = writeln!( out, " case {opcode}: return {}(self, by_argv, 1, NULL);", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } // a comparison the class does not define is not an error: answering @@ -2379,7 +2559,7 @@ fn ctype(module: &ModuleIr, ty: &RType) -> String { // a class whose fields sit past a base's instance is *not* its field struct: // the two addresses differ, and only the object pointer identifies the value Some(owner) if !external_storage(module, owner) => { - format!("{} *", owner.struct_name(&module.name)) + format!("{} *", owner.struct_name(module.name.dotted())) } _ => "PyObject *".to_string(), }, @@ -2420,7 +2600,7 @@ fn unbox_checked(module: &ModuleIr, ty: &RType, expr: &str) -> String { Some(owner) => format!( "({})By_UnboxInstance({expr}, (PyTypeObject *){}_OBJ)", ctype(module, ty), - owner.type_name(&module.name) + owner.type_name(module.name.dotted()) ), // a class with no emitted layout is represented as a plain object None => unbox_expr(ty, expr), @@ -2486,7 +2666,7 @@ fn signature(module: &ModuleIr, function: &Function) -> String { format!( "static {} {}({})", ctype(module, &function.ret), - function.native_symbol(&module.name), + function.native_symbol(module.name.dotted()), params ) } @@ -2873,6 +3053,42 @@ fn commit_checked(function: &Function, dest: RegisterId, error_target: Option String, + error_target: Option, +) -> String { + let slot = format!("by_g_{}", mangle(name)); + let mut out = format!(" {{ static PyObject *{slot} = NULL;\n"); + let _ = writeln!( + out, + " if ({slot} == NULL) {slot} = By_InternedStr({});", + c_string_sized(name) + ); + let _ = writeln!( + out, + " if ({slot} == NULL) goto {};", + error_label(error_target) + ); + out.push_str(&assign_checked( + module, + function, + dest, + &call(&slot), + error_target, + )); + out.push_str(" }\n"); + out +} + /// the tests that turn a read of an unwritten local into `UnboundLocalError` /// /// every read of a flagged register is guarded, not only the ones the analysis found @@ -3366,8 +3582,8 @@ fn emit_op( .collect::>() .join(", "); let symbol = match target { - Some(target) => target.native_symbol(&module.name), - None => format!("by_{}_{}", mangle(&module.name), mangle(callee)), + Some(target) => target.native_symbol(module.name.dotted()), + None => format!("by_{}_{}", mangle(module.name.dotted()), mangle(callee)), }; let call = format!("{symbol}({args})"); let fallible = target.is_none_or(|target| target.convention.can_fail()); @@ -3695,19 +3911,29 @@ fn emit_op( else { return String::new(); }; - let struct_name = owner.struct_name(&module.name); - let type_name = owner.type_name(&module.name); + let struct_name = owner.struct_name(module.name.dotted()); + let type_name = owner.type_name(module.name.dotted()); + // `tp_alloc` answers with the *object*, which is only the field storage for + // a class that owns its layout. one appending to a base keeps its fields + // past that base's instance, so the two addresses differ and `fields_of` is + // what knows by how much — writing through the object pointer would land on + // the base's own data let mut out = format!( " {{ PyTypeObject *by_type = (PyTypeObject *){type_name}_OBJ;\n\ - \x20 {struct_name} *by_new = ({struct_name} *)by_type->tp_alloc(by_type, 0);\n" + \x20 PyObject *by_obj = by_type->tp_alloc(by_type, 0);\n" ); // `tp_alloc` zeroes the block, so a field the loop below misses is NULL // rather than garbage — but every field is written let _ = writeln!( out, - " if (by_new == NULL) goto {};", + " if (by_obj == NULL) goto {};", error_label(error_target) ); + let _ = writeln!( + out, + " {{ {struct_name} *by_new = {};", + fields_of(module, owner, "by_obj") + ); for (field, value) in owner.fields.iter().zip(fields) { // `tp_alloc` zeroes the block, so `None` leaves the field NULL — // which is what an unset cell is @@ -3727,13 +3953,14 @@ fn emit_op( let _ = writeln!(out, " by_new->{} = 1;", field.presence()); } } + out.push_str(" }\n"); if let Some(release) = dec_ref(&RType::OBJECT, &local(*dest)) { let _ = writeln!(out, " {release}"); } let destination = function .register(*dest) .map_or_else(|| "PyObject *".to_string(), |decl| ctype(module, &decl.ty)); - let _ = writeln!(out, " {} = ({destination})by_new; }}", local(*dest)); + let _ = writeln!(out, " {} = ({destination})by_obj; }}", local(*dest)); out } Op::Enter { dest, manager } => { @@ -3832,7 +4059,7 @@ fn emit_op( else { return String::new(); }; - let table = format!("{}_methods", owner.type_name(&module.name)); + let table = format!("{}_methods", owner.type_name(module.name.dotted())); let call = format!( "By_MakeClosure(&{table}[{index}], (PyObject *)({}))", value_expr(env) @@ -3860,6 +4087,29 @@ fn emit_op( out.push_str(&commit_checked(function, *dest, error_target)); out } + // the write half of `LoadGlobal`, reaching the same dict through the same + // interned key — a register write would leave the module's binding alone + Op::StoreGlobal { dest, name, value } => global_namespace_op( + module, + function, + *dest, + name, + &|slot| { + format!( + "By_StoreGlobal(by_module_dict, {slot}, {})", + value_expr(value) + ) + }, + error_target, + ), + Op::DeleteGlobal { dest, name } => global_namespace_op( + module, + function, + *dest, + name, + &|slot| format!("By_DeleteGlobal(by_module_dict, {slot})"), + error_target, + ), Op::LoadClass { dest, class } => { let Some(owner) = module .classes @@ -3868,7 +4118,7 @@ fn emit_op( else { return String::new(); }; - let type_name = owner.type_name(&module.name); + let type_name = owner.type_name(module.name.dotted()); assign_owned( module, function, @@ -4508,7 +4758,7 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> String { let mut out = format!( "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n", - function.wrapper_symbol(&module.name) + function.wrapper_symbol(module.name.dotted()) ); // a method's receiver arrives in `self`, so it does not count as an argument if !is_method { @@ -4621,7 +4871,7 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri out, " {} by_result = {}({args});", ctype(module, &function.ret), - function.native_symbol(&module.name) + function.native_symbol(module.name.dotted()) ); if function.convention.can_fail() { let _ = writeln!( @@ -4652,22 +4902,22 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri // the jump here happens before any argument local is filled, so there is // nothing to release — and no error is set, because nothing went wrong if !function.deferring.is_empty() || !function.computed_defaults.is_empty() { + // the twin is taken off the interpreted class, and for a *static* method that + // is the plain function the `staticmethod` wraps — so the call is the one a + // module-level function makes, and `self` holds nothing to put in front of it + let hands_over_self = function.owner.is_some() && function.binding != Binding::Static; let _ = writeln!(out, "by_wrap_defer: ;"); let _ = writeln!( out, " return {}({}, {}, {}args, nargs, kwnames);", - if function.owner.is_some() { + if hands_over_self { "By_CallInterpretedMethod" } else { "By_CallInterpreted" }, - function.interpreted_symbol(&module.name), + function.interpreted_symbol(module.name.dotted()), c_string(&function.name), - if function.owner.is_some() { - "self, " - } else { - "" - } + if hands_over_self { "self, " } else { "" } ); } out.push_str("}\n"); @@ -4814,6 +5064,8 @@ fn external_construction( class: &ClassIr, type_name: &str, pack: &str, + slot: Option, + twins: usize, ) -> String { let spec = if !class.keywords.is_empty() || stands_on_an_emitted_base(module, class) { // a keyword has nowhere to go in a spec, and a base of ours beside one from @@ -4827,22 +5079,68 @@ fn external_construction( } else { "by_kwds" }; + // the constants a metaclass construction writes into the namespace, and reads back off + // the class to see whether it agreed. a class no interpreted `class` statement wrote + // has no captured body to take a value off, and there is nothing to carry + let (declare, constants) = match slot.filter(|_| !class.constants.is_empty()) { + None => (String::new(), "NULL".to_string()), + Some(slot) => { + let names = class + .constants + .iter() + .map(|name| c_string(name)) + .collect::>() + .join(", "); + ( + format!( + "\x20 static const char *const by_constants[] = {{{names}}};\n\ + \x20 By_ClassConstants by_carried = {{by_body[{slot}], by_constants, {}, by_twin, by_type, {twins}}};\n", + class.constants.len() + ), + "&by_carried".to_string(), + ) + } + }; let build = format!( - "By_BuildClass(dict, {}, {pack}, {keywords}, {type_name}_methods, {spec}, {})", + "By_BuildClass(dict, {}, {pack}, {keywords}, {type_name}_methods, {spec}, {}, {constants})", c_string(&class.name), i32::from(metaclass_construction(class)) ); if class.keywords.is_empty() { - return format!("\x20 {type_name} = {build};\n"); + return format!("{declare}\x20 {type_name} = {build};\n"); } format!( - "\x20 {{ PyObject *by_kwds = {type_name}_keywords(dict);\n\ + "{declare}\ + \x20 {{ PyObject *by_kwds = {type_name}_keywords(dict);\n\ \x20 if (by_kwds == NULL) return -1;\n\ \x20 {type_name} = {build};\n\ \x20 Py_DECREF(by_kwds); }}\n" ) } +/// every decorator this module's init applies, applied where the twin left off +/// +/// the source the twin runs has these taken out of it — see +/// [`ModuleIr::decorated_at_init`](by_ir::function::ModuleIr::decorated_at_init) — so an +/// init that gives up before it has installed anything of its own still has to run them, +/// or the module is left holding definitions nothing ever decorated. it applies them to +/// the namespace entry, which on that path is still the twin's own definition — which is +/// exactly where python would have applied them +fn twin_decorators(module: &ModuleIr) -> String { + let mut out = String::new(); + for decoration in module.decorated_at_init() { + for decorator in decoration.decorators.iter().rev() { + let _ = writeln!( + out, + " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", + c_string(decoration.name), + c_string(&decorator.dotted()) + ); + } + } + out +} + fn emit_module_init(module: &ModuleIr) -> String { let mut out = String::new(); @@ -4861,26 +5159,38 @@ fn emit_module_init(module: &ModuleIr) -> String { out, " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS, NULL}},", function.name, - function.wrapper_symbol(&module.name) + function.wrapper_symbol(module.name.dotted()) ); } } out.push_str(" {NULL, NULL, 0, NULL}\n};\n\n"); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + // the `PyModuleDef`'s `m_name`, which is not where a module's `__name__` comes + // from: this is a multi-phase init, so python builds the module from the + // *spec*'s name and never reads this one. it says the last component because + // that is what the init symbol beside it is named after + let last = module.name.last_component(); // `m_methods` is NULL and the natives are installed from the exec slot // instead, so they land *after* the interpreted definitions rather than // being overwritten by them // decorators run last: the native function has to be in the namespace before - // a decorator can be applied to it + // a decorator can be applied to it. + // + // `exported` is what says there is a namespace entry to apply one to at all — an + // unboxed edition is a second function under a mangled name nothing binds, and + // reaching for it here would fail the import with a `NameError`. it is also the + // condition `ModuleIr::decorated_at_init` states, and the twin's source has these + // decorators taken out of it on the strength of that: applying one here that the + // twin no longer applies, or the reverse, is what makes a decorator run twice or + // not at all let mut decorators = String::new(); - for function in &module.functions { + for function in module.functions.iter().filter(|function| function.exported) { for decorator in function.decorators.iter().rev() { let _ = writeln!( decorators, " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", c_string(&function.name), - c_string(decorator) + c_string(&decorator.dotted()) ); } } @@ -4892,20 +5202,83 @@ fn emit_module_init(module: &ModuleIr) -> String { // base's do. so a refusal is a whole-module one — the interpreted definition already // built the module, and it is left standing rather than made into a half-native // mixture + let mut conditions = Vec::new(); + // below 3.12 there is no way to say where appended storage goes at all, so no such + // class has a construction and the module has none either + if module + .classes + .iter() + .any(|class| appends_storage_from_a_spec(module, class)) + { + conditions.push("!BY_HAS_TYPE_DATA"); + } + // a class keeping an instance dict is the same question again: below 3.13 there is + // no published way to walk or release a managed one, and a collected type that + // cannot walk what it holds is worse than no compiled type at all + if module + .classes + .iter() + .any(|class| instance_dict(module, class)) + { + conditions.push("!BY_HAS_MANAGED_DICT"); + } + // whether the fallback source has to be run with its class bodies captured. the + // capture costs a dict copy per class the body writes, so a module with no constant + // to carry runs its body the plain way + let captures_bodies = module + .classes + .iter() + .any(|class| class.exported && !class.constants.is_empty()); + let release_bodies = if captures_bodies { + " Py_XDECREF(by_bodies);\n" + } else { + "" + }; let mut layout_guard = String::new(); + if !conditions.is_empty() { + // the twin's source no longer carries the decorators init applies, so leaving + // the module interpreted means applying them here — to the twin's own + // definitions, which is where python would have run them + let _ = write!( + layout_guard, + " if ({}) {{\n{}{release_bodies} return 0;\n }}\n", + conditions.join(" || "), + twin_decorators(module) + ); + } for class in &module.classes { if appends_storage_from_a_spec(module, class) { - let type_name = class.type_name(&module.name); - if layout_guard.is_empty() { - // below 3.12 there is no way to say where appended storage goes at all, - // so no such class has a construction and the module has none either - layout_guard.push_str(" if (!BY_HAS_TYPE_DATA) return 0;\n"); - } + let type_name = class.type_name(module.name.dotted()); + // one of these standing on another is built on the *finished* type below it + // rather than on the interpreted definition — which is the only base such a + // class can chain a deallocation to. the order is the module's, so the one + // below is already built + let construction = match appended_over_an_emitted_base(module, class) { + Some(base) => format!( + "By_SpecSubclass(dict, {}, &{type_name}_spec, {}, {}_OBJ)", + c_string(&class.name), + c_string(&base.name), + base.type_name(module.name.dotted()) + ), + None => format!( + "By_SpecClass(dict, {}, &{type_name}_spec)", + c_string(&class.name) + ), + }; + // giving up here leaves every interpreted definition standing, and their + // decorators have been taken out of the source that built them — so this + // exit has to run them for the same reason the guard above does. a module + // with nothing to run keeps the plain one-line refusal + let unwind = format!("{}{release_bodies}", twin_decorators(module)); + let refusal = if unwind.is_empty() { + "return 0;".to_string() + } else { + format!("{{\n{unwind} return 0;\n }}") + }; let _ = writeln!( layout_guard, - " {type_name} = By_SpecClass(dict, {}, &{type_name}_spec);\n\ - \x20 if ({type_name} == NULL) return 0;", - c_string(&class.name) + " {type_name} = {construction};\n\ + \x20 if ({type_name} == NULL) {refusal}" ); } } @@ -4925,9 +5298,35 @@ fn emit_module_init(module: &ModuleIr) -> String { .collect(); let mut twin_init = String::new(); let mut adopt_init = String::new(); + // and the alias remap after that, once every decorator has settled what stands under + // each class's own name — see `By_RemapTwinAliases` for why it waits that long + let mut twin_remap = String::new(); if !twins.is_empty() { let count = twins.len(); - let _ = writeln!(twin_init, " PyObject *by_twin[{count}];"); + // the types are held alongside the twins from here rather than gathered at the + // adoption, because a class constant is remapped against them as its class is + // built. a slot is NULL until then, and `By_TwinReplacement` reads that as a + // refusal — so a constant naming a class built later is left off rather than + // copied across as the twin + let _ = writeln!( + twin_init, + " PyObject *by_twin[{count}];\n\ + \x20 PyObject *by_type[{count}] = {{NULL}};" + ); + // and the bodies those `class` statements wrote, which is where a class-level + // constant's value comes from — the twin has been through its own decorators by + // now, and `By_RunModuleBody` says what that costs. borrowed from `by_bodies`, + // which is held for the whole of this function + if twins.iter().any(|class| !class.constants.is_empty()) { + let _ = writeln!(twin_init, " PyObject *by_body[{count}];"); + for (slot, class) in twins.iter().enumerate() { + let _ = writeln!( + twin_init, + " by_body[{slot}] = By_ClassBody(by_bodies, {});", + c_string(&class.name) + ); + } + } for (slot, class) in twins.iter().enumerate() { let _ = writeln!( twin_init, @@ -4937,27 +5336,44 @@ fn emit_module_init(module: &ModuleIr) -> String { } let _ = writeln!( adopt_init, - " {{ PyObject *by_type[{count}];\n\ - \x20 int by_carried;" + " if (By_AdoptTwinAttributes(by_twin, by_type, {count}) < 0) return -1;" ); - for (slot, class) in twins.iter().enumerate() { - let _ = writeln!( - adopt_init, - "\x20 by_type[{slot}] = {}_OBJ;", - class.type_name(&module.name) - ); - } + let names = twins + .iter() + .map(|class| c_string(&class.name)) + .collect::>() + .join(", "); let _ = writeln!( - adopt_init, - "\x20 by_carried = By_AdoptTwinAttributes(by_twin, by_type, {count});\n\ + twin_remap, + " {{ static const char *const by_name[] = {{{names}}};\n\ + \x20 int by_remapped = By_RemapTwinAliases(dict, by_twin, by_name, {count});\n\ \x20 for (Py_ssize_t by_at = 0; by_at < {count}; by_at++) Py_XDECREF(by_twin[by_at]);\n\ - \x20 if (by_carried < 0) return -1; }}" + \x20 if (by_remapped < 0) return -1; }}" ); } let mut class_init = String::new(); + // a class decorator is arbitrary python handed the class, and what it reads is the + // class *body* — its annotations, its class-level defaults, what its `__dict__` + // holds. none of that is on the emitted type until the twin's attributes have been + // adopted, so every decorator waits for that and runs in a pass of its own. + // + // what this costs is a class resolving an in-module base *through the namespace* — + // the mixed `class C(Base, Mixin)` shape — which now stands on the emitted type + // rather than on what `Base`'s decorator returned. that is already what a class on + // an in-module base alone does, which builds on `Base`'s type object directly + let mut class_decorate = String::new(); + // which slot of the twin arrays a class occupies. it is counted rather than looked up + // by name, because `twins` is this same walk filtered on `exported` and two classes in + // one module can be given the same name + let mut exported_so_far = 0; for class in &module.classes { - let type_name = class.type_name(&module.name); + let type_name = class.type_name(module.name.dotted()); + let slot = class.exported.then(|| { + let slot = exported_so_far; + exported_so_far += 1; + slot + }); let ready = if appends_storage_from_a_spec(module, class) { // already built, by the one construction open to it String::new() @@ -4987,7 +5403,12 @@ fn emit_module_init(module: &ModuleIr) -> String { module, class, &type_name, - &format!("PyTuple_Pack(1, {}_OBJ)", base.type_name(&module.name)) + &format!( + "PyTuple_Pack(1, {}_OBJ)", + base.type_name(module.name.dotted()) + ), + slot, + twins.len() ) ) } else if let Some(externals) = class.base.as_ref().and_then(ClassBase::external) { @@ -5033,7 +5454,7 @@ fn emit_module_init(module: &ModuleIr) -> String { {}\ \x20 {releases}\n\ \x20 if ({type_name} == NULL) return -1; }}", - external_construction(module, class, &type_name, &pack) + external_construction(module, class, &type_name, &pack, slot, twins.len()) ) } else { format!( @@ -5044,6 +5465,12 @@ fn emit_module_init(module: &ModuleIr) -> String { if !ready.is_empty() { let _ = writeln!(class_init, "{ready}"); } + // the type exists from here, so it is what stands for this class's twin in every + // remap below. a class built in the layout guard was built before the array even + // existed, which is why this is not written where the construction is + if let Some(slot) = slot { + let _ = writeln!(class_init, " by_type[{slot}] = {type_name}_OBJ;"); + } // the awaitable `__anext__` hands back is a type of its own, and an unreadied // type has no `tp_free` — `PyObject_New` on one segfaults rather than failing if class @@ -5072,11 +5499,13 @@ fn emit_module_init(module: &ModuleIr) -> String { // evaluated it at class-definition time. copying keeps the *same* object, // which is what evaluating once means for constant in &class.constants { + // a class no interpreted `class` statement wrote has no body to take one off + let Some(slot) = slot else { continue }; let _ = writeln!( class_init, - " if (By_CopyClassConstant(dict, {}, (PyTypeObject *){type_name}_OBJ, {}) < 0) return -1;", - c_string(&class.name), - c_string(constant) + " if (By_CopyClassConstant(by_body[{slot}], (PyTypeObject *){type_name}_OBJ, {}, by_twin, by_type, {}) < 0) return -1;", + c_string(constant), + twins.len() ); } // a decorated method is decorated *after* the type exists, which is the only @@ -5090,7 +5519,7 @@ fn emit_module_init(module: &ModuleIr) -> String { let names = method .decorators .iter() - .map(|decorator| c_string(decorator)) + .map(|decorator| c_string(&decorator.dotted())) .collect::>() .join(", "); let _ = writeln!( @@ -5110,14 +5539,16 @@ fn emit_module_init(module: &ModuleIr) -> String { " if (PyDict_SetItemString(dict, \"{}\", {type_name}_OBJ) < 0) return -1;", class.name ); - // and its own decorators after that, because a decorator replaces the - // namespace entry — which is where every construction looks + // and its own decorators after the adoption, because a decorator *reads* the + // class it is handed and every one of those reads has to see the body the + // `class` statement wrote. they go after the namespace entry too, because a + // decorator replaces it — which is where every construction looks for decorator in class.decorators.iter().rev() { let _ = writeln!( - class_init, + class_decorate, " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", c_string(&class.name), - c_string(decorator) + c_string(&decorator.dotted()) ); } } @@ -5143,7 +5574,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // taken while the interpreted definition is still the one under this name let mut interpreted_init = String::new(); for function in module.all_functions().filter(|function| function.defers()) { - let handle = function.interpreted_symbol(&module.name); + let handle = function.interpreted_symbol(module.name.dotted()); match &function.owner { // a method's twin is an attribute of the interpreted class, which is what // sits under the class's name until the compiled type replaces it @@ -5169,6 +5600,27 @@ fn emit_module_init(module: &ModuleIr) -> String { } } + // the body is run either way; what a class-level constant needs is what each `class` + // statement wrote *before* its own decorators, which only the capturing run keeps + let run_body = if captures_bodies { + "\x20 PyObject *by_bodies = NULL;\n\ + \x20 if (by_fallback_source[0] != '\\0') {\n\ + \x20 by_bodies = By_RunModuleBody(by_fallback_source, dict);\n\ + \x20 if (by_bodies == NULL) return -1;\n\ + \x20 }\n" + .to_string() + } else { + "\x20 if (by_fallback_source[0] != '\\0') {\n\ + \x20 if (PyDict_GetItemString(dict, \"__builtins__\") == NULL &&\n\ + \x20 PyDict_SetItemString(dict, \"__builtins__\", PyEval_GetBuiltins()) < 0) {\n\ + \x20 return -1;\n\ + \x20 }\n\ + \x20 PyObject *result = PyRun_String(by_fallback_source, Py_file_input, dict, dict);\n\ + \x20 if (result == NULL) return -1;\n\ + \x20 Py_DECREF(result);\n\ + \x20 }\n" + .to_string() + }; let _ = write!( out, "static int by_exec(PyObject *module) {{\n\ @@ -5176,20 +5628,15 @@ fn emit_module_init(module: &ModuleIr) -> String { \x20 if (dict == NULL) return -1;\n\ \x20 by_module_dict = dict;\n\ {literal_init}\ - \x20 if (by_fallback_source[0] != '\\0') {{\n\ - \x20 if (PyDict_GetItemString(dict, \"__builtins__\") == NULL &&\n\ - \x20 PyDict_SetItemString(dict, \"__builtins__\", PyEval_GetBuiltins()) < 0) {{\n\ - \x20 return -1;\n\ - \x20 }}\n\ - \x20 PyObject *result = PyRun_String(by_fallback_source, Py_file_input, dict, dict);\n\ - \x20 if (result == NULL) return -1;\n\ - \x20 Py_DECREF(result);\n\ - \x20 }}\n\ + {run_body}\ {layout_guard}\ {interpreted_init}\ {twin_init}\ {class_init}\ {adopt_init}\ + {class_decorate}\ + {twin_remap}\ + {release_bodies}\ \x20 if (PyModule_AddFunctions(module, by_methods) < 0) return -1;\n\ {decorators}\ \x20 return 0;\n\ @@ -5232,7 +5679,7 @@ mod tests { fn module_with(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), @@ -5415,6 +5862,227 @@ mod tests { assert!(refusal < install, "the refusal comes first: {c}"); } + /// a class appending storage: `tp_alloc` hands back the *object*, and its fields are + /// somewhere past it. writing them through the object pointer would land on the + /// base's own data — 8 bytes into a `complex`, say — so the construction has to + /// reach the storage the same way every other field access does + #[test] + fn constructing_a_class_that_appends_storage_writes_past_the_base() { + let mut builder = FunctionBuilder::new("make", RType::OBJECT); + let tag = builder.param("tag", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + builder.push(Op::NewInstance { + dest: made, + class: "Wrapped".to_string(), + fields: vec![Some(Value::Register(tag))], + }); + builder.terminate(Terminator::Return(Value::Register(made))); + let mut module = module_with(builder.finish()); + module.classes.push(appending_class()); + let c = emit_module(&module); + + assert!( + c.contains( + "By_app_Wrapped *by_new = ((By_app_Wrapped *)By_TypeData(by_obj, By_app_Wrapped_Type_OBJ));" + ), + "the storage is found, not assumed: {c}" + ); + assert!( + !c.contains("(By_app_Wrapped *)by_type->tp_alloc"), + "the allocation is an object, not a field struct: {c}" + ); + // and the value handed on is still the object — the field storage is not one + assert!(c.contains("r1 = (PyObject *)by_obj;"), "{c}"); + } + + /// the same construction for a class that owns its layout: the object *is* the + /// storage there, and finding it must not cost a lookup + #[test] + fn constructing_a_class_that_owns_its_layout_casts_the_object() { + let mut builder = FunctionBuilder::new("make", RType::OBJECT); + let tag = builder.param("tag", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + builder.push(Op::NewInstance { + dest: made, + class: "Owned".to_string(), + fields: vec![Some(Value::Register(tag))], + }); + builder.terminate(Terminator::Return(Value::Register(made))); + let mut module = module_with(builder.finish()); + let mut owned = appending_class(); + owned.name = "Owned".to_string(); + owned.base = None; + module.classes.push(owned); + let c = emit_module(&module); + + assert!( + c.contains("By_app_Owned *by_new = (By_app_Owned *)by_obj;"), + "no lookup for a class that owns its layout: {c}" + ); + assert!(!c.contains("By_TypeData(by_obj"), "{c}"); + } + + /// two classes appending storage, one over the other. the inner one is built on the + /// *finished* type of the outer rather than on the interpreted definition, because + /// the interpreted definition's `tp_dealloc` is `subtype_dealloc` — which picks the + /// deallocator to chain to out of `Py_TYPE(self)`, finds this class's own and calls + /// it back until the stack runs out + #[test] + fn appended_storage_over_appended_storage_stands_on_the_emitted_base() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields[0].name = "depth".to_string(); + module.classes.push(inner); + let c = emit_module(&module); + + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_app_Deeper_Type_spec, \"Wrapped\", By_app_Wrapped_Type_OBJ);" + ), + "the subclass stands on the emitted base: {c}" + ); + // the base is still the one construction that has to answer from reality, and + // it is built first + let base = c + .find("By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\"") + .expect("the base is built from the twin's own bases"); + let derived = c + .find("By_app_Deeper_Type = By_SpecSubclass(") + .expect("the subclass is built after it"); + assert!(base < derived, "the base is built first: {c}"); + // and a refusal is still the whole module's + assert!( + c.contains("if (By_app_Deeper_Type == NULL) return 0;"), + "{c}" + ); + } + + /// an instance counts as one reference to its type however deep the chain of + /// appended storage is. each traverse chains to the one below, so reporting the link + /// unconditionally would report it once per rung — and a collector told an instance + /// holds two references where it holds one has been told the type is garbage + #[test] + fn the_link_to_the_type_is_reported_by_exactly_one_traverse() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let c = emit_module(&module); + assert!( + c.contains( + " if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_VISIT(Py_TYPE(self));" + ), + "the rung whose base carries it already does not: {c}" + ); + assert_eq!( + c.matches("Py_VISIT(Py_TYPE(self))").count(), + 1, + "one report per traverse: {c}" + ); + } + + /// and the same rung drops it. an instance holds one reference to its type; a chain + /// of appended storage whose every rung drops one loses the type after five + /// instances and the process goes with it + #[test] + fn the_reference_to_the_type_is_dropped_by_exactly_one_deallocator() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let c = emit_module(&module); + assert!( + c.contains( + " if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)\n\ + \x20 && (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_DECREF(by_type);" + ), + "the rung whose base drops it already does not: {c}" + ); + assert_eq!(c.matches("Py_DECREF(by_type)").count(), 1, "{c}"); + // and the base is read before the deallocation, because after it the object is + // gone and the type pointer with it + let read = c.find("by_base = ").expect("the base is read"); + let free = c + .find("by_base->tp_dealloc(self);") + .expect("and chained to"); + assert!(read < free, "{c}"); + } + + /// the other layout model for an in-module base: the subclass restates its base's + /// fields so that a pointer to one is a pointer to the other. an appended region + /// would give the pair two copies of each, so such a class is not appended over + /// anything and keeps the construction that answers from reality — which refuses + #[test] + fn a_subclass_restating_its_bases_fields_is_not_appended_over_it() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields.push(by_ir::function::FieldDecl { + name: "depth".to_string(), + ty: RType::OBJECT, + default: None, + optional: false, + }); + module.classes.push(inner); + let c = emit_module(&module); + + assert!(!c.contains("By_SpecSubclass"), "{c}"); + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_app_Deeper_Type_spec);" + ), + "{c}" + ); + } + + /// a base that lays nothing out of its own is built by calling its metaclass, so its + /// type is a `class` statement's after all — `subtype_dealloc` and the recursion + /// that comes with it. only a base built from a spec here can be chained to + #[test] + fn a_base_this_module_does_not_build_from_a_spec_is_not_one_to_stand_on() { + let mut module = module_with(add()); + let mut base = appending_class(); + base.fields.clear(); + module.classes.push(base); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields[0].name = "depth".to_string(); + module.classes.push(inner); + let c = emit_module(&module); + + assert!(!c.contains("By_SpecSubclass"), "{c}"); + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_app_Deeper_Type_spec);" + ), + "{c}" + ); + } + + fn appending_class() -> ClassIr { + ClassIr { + name: "Wrapped".to_string(), + immutable: false, + exported: true, + base: Some(ClassBase::External(vec!["Exception".to_string()])), + inherited_init: false, + generic: false, + constants: Vec::new(), + fields: vec![by_ir::function::FieldDecl { + name: "tag".to_string(), + ty: RType::OBJECT, + default: None, + optional: false, + }], + decorators: Vec::new(), + methods: Vec::new(), + resume: None, + keywords: Vec::new(), + } + } + #[test] fn an_infallible_function_emits_no_error_path_at_all() { // this is error-path elision: the `raises Never` contract, in the C @@ -5444,7 +6112,7 @@ mod tests { caller.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![callee.finish(), caller.finish()], declined: Vec::new(), classes: Vec::new(), @@ -5634,7 +6302,7 @@ mod tests { #[test] fn the_module_init_matches_the_import_name() { let module = ModuleIr { - name: "pkg.app".to_string(), + name: by_ir::ModuleName::new("pkg.app"), functions: vec![add()], classes: Vec::new(), gradual: Vec::new(), @@ -5655,6 +6323,38 @@ mod tests { )); } + /// cpython reads a class's `__module__` off the front of its `tp_name` and its + /// `__name__` off the back, so a class in a package needs the *whole* dotted + /// name there. emitting only the last component made a class in `tkinter/m.py` + /// answer `__module__ == "m"`, which `sys.modules` has nothing under + #[test] + fn a_type_in_a_package_names_the_dotted_module_it_came_from() { + let mut module = module_with(add()); + module.name = by_ir::ModuleName::new("pkg.app"); + module.classes.push(ClassIr { + name: "Point".to_string(), + immutable: false, + exported: true, + base: None, + inherited_init: false, + generic: false, + constants: Vec::new(), + fields: vec![by_ir::function::FieldDecl { + name: "x".to_string(), + ty: RType::INT, + default: None, + optional: false, + }], + decorators: Vec::new(), + methods: Vec::new(), + resume: None, + keywords: Vec::new(), + }); + let c = emit_module(&module); + assert!(c.contains("\"pkg.app.Point\""), "{c}"); + assert!(!c.contains("\"app.Point\""), "{c}"); + } + #[test] fn the_interpreted_definitions_run_before_the_natives_are_installed() { let mut module = module_with(add()); diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index e11685f515..d70f894c70 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -5,7 +5,7 @@ //! and refuses to finish a block twice, which removes the two mistakes that would //! otherwise account for most verifier failures. -use crate::function::{BasicBlock, CallConvention, Function, RegisterDecl}; +use crate::function::{BasicBlock, CallConvention, Decorator, Function, RegisterDecl}; use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; use crate::rtype::RType; @@ -17,7 +17,7 @@ pub struct FunctionBuilder { convention: CallConvention, exported: bool, owner: Option, - decorators: Vec, + decorators: Vec, registers: Vec, /// `None` until the block is sealed with a terminator blocks: Vec>, @@ -78,7 +78,7 @@ impl FunctionBuilder { } /// decorators to apply at module init, outermost first - pub fn decorators(&mut self, decorators: Vec) -> &mut Self { + pub fn decorators(&mut self, decorators: Vec) -> &mut Self { self.decorators = decorators; self } @@ -276,6 +276,7 @@ impl FunctionBuilder { range: self.range, deferring: self.deferring, computed_defaults: self.computed_defaults, + binding: crate::function::Binding::Instance, } } } diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index f2d0d8d57c..63404beb54 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -4,6 +4,8 @@ //! declared once with a type, which is what makes the representation invariant //! checkable: every write to a register must produce that register's type. +use std::path::PathBuf; + use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; use crate::rtype::RType; @@ -107,6 +109,103 @@ impl CallConvention { } } +/// a decorator expression, as much of one as module init can evaluate +/// +/// python evaluates the expression where the `def` or the `class` stands, in the +/// enclosing frame. module-level code is not compiled — it runs from the interpreted +/// twin — so there is no native frame standing there, and the only moment left is +/// module init, once that twin has run the whole body and the native definition is in +/// the namespace. +/// +/// that is faithful for an expression which is nothing but *reads*, and only for one. +/// a call would be made a second time, at the end of the module rather than where it +/// was written, and whatever it did on the way would happen twice — so `@lru_cache(512)` +/// has no variant here and declines instead +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decorator { + /// a name, or a chain of attributes read off one: `functools.cache` + /// + /// the root resolves the way `LOAD_GLOBAL` resolves it, so a decorator defined in + /// the module, imported into it, or a builtin all work; each attribute is then a + /// plain `getattr` + Path { + root: String, + attributes: Vec, + }, +} + +impl Decorator { + /// a decorator written as a bare name — which is what every language *modifier* + /// translates to + pub fn name(name: impl Into) -> Self { + Self::Path { + root: name.into(), + attributes: Vec::new(), + } + } + + /// the bare name this is, where it is one + pub fn as_name(&self) -> Option<&str> { + match self { + Self::Path { root, attributes } if attributes.is_empty() => Some(root), + Self::Path { .. } => None, + } + } + + /// the name the resolution starts from, whatever shape the rest is + pub fn root(&self) -> &str { + match self { + Self::Path { root, .. } => root, + } + } + + /// the expression as written, which is also how codegen spells it: a python + /// identifier holds no `.`, so the join is unambiguous to split again + pub fn dotted(&self) -> String { + match self { + Self::Path { root, attributes } if attributes.is_empty() => root.clone(), + Self::Path { root, attributes } => format!("{root}.{}", attributes.join(".")), + } + } +} + +/// what python leaves in slot zero when a method is called through the class +/// +/// the three conventions `type_add_methods` distinguishes, and the reason they cannot +/// be inferred from the parameter list: `def make(x)` under `@staticmethod` takes an +/// `x`, and the same source without it takes a receiver written `x`. a module-level +/// function is [`Instance`](Self::Instance) and nothing reads it there — the field +/// only means anything about a method +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Binding { + /// slot zero holds the receiver: the ordinary method + #[default] + Instance, + /// there is no receiver, and slot zero is the first written parameter + Static, + /// slot zero holds the *class* rather than an instance of it + Class, +} + +impl Binding { + /// whether the boundary is handed slot zero outside the argument vector + /// + /// true for a class method as much as an instance one: `METH_CLASS` puts the class + /// in `self`, which is the same slot in the same place + pub fn takes_slot_zero_from_self(self) -> bool { + matches!(self, Self::Instance | Self::Class) + } + + /// the `METH_` flag this convention adds to a method table entry + pub fn method_flag(self) -> Option<&'static str> { + match self { + Self::Instance => None, + Self::Static => Some("METH_STATIC"), + Self::Class => Some("METH_CLASS"), + } + } +} + /// a compiled function #[derive(Debug, Clone, PartialEq)] pub struct Function { @@ -173,11 +272,14 @@ pub struct Function { /// not the arity pub computed_defaults: Vec, /// decorators to apply, outermost first, after the native function is - /// installed in the module namespace. + /// installed in the module namespace + pub decorators: Vec, + /// what python puts in slot zero when this method is reached through the class. /// - /// each is a name resolved the way `LOAD_GLOBAL` resolves it, so a decorator - /// defined in the module or imported into it both work - pub decorators: Vec, + /// `staticmethod` and `classmethod` are honoured by the method table entry rather + /// than by [applying the decorator](Self::decorators) at init — the decorator is + /// dropped where this says anything other than [`Binding::Instance`] + pub binding: Binding, } impl Function { @@ -448,7 +550,7 @@ pub struct ClassIr { /// a construction resolves the class through that namespace, so it gets whatever /// the decorator produced — which is what makes decorating the *class* sound /// where decorating a construction site would not be - pub decorators: Vec, + pub decorators: Vec, /// methods, whose first parameter is the receiver pub methods: Vec, /// how the class resumes, when it is a generator or a coroutine @@ -544,8 +646,8 @@ impl LineTable { /// a module's worth of compiled functions #[derive(Debug, Clone, PartialEq)] pub struct ModuleIr { - /// the dotted module name, as python will import it - pub name: String, + /// the module's name — see [`ModuleName`] for why that is not one string + pub name: ModuleName, pub functions: Vec, pub classes: Vec, /// functions the compiler declined, with the reason. these fall back to the @@ -615,7 +717,65 @@ pub fn qualify(owner: Option<&str>, name: &str) -> String { } } +/// a module-level definition whose decorators module init applies to the native one +/// +/// python evaluates a decorator where the `def` or the `class` stands. the interpreted +/// twin is what stands there, so the twin evaluates it — and module init then evaluates +/// it a second time, over the native definition that replaces the twin's. the binding +/// the namespace ends up with is right either way, so what shows is not a wrong value +/// but a side effect that happened twice: a registry with two entries for one function. +/// +/// the decorator is therefore taken out of the source the twin runs, and this is the one +/// description of which ones: it drives both that removal and the C that applies them, so +/// the two cannot come to disagree. +/// +/// a module-level *function* and a module-level *class* are both here. a class the +/// module body still reaches before init has run does not get this far: it declines +/// instead, because between the twin's `class` statement and init the name holds an +/// undecorated definition, and whatever read it in that window keeps what it read +/// +/// a *method* is deliberately not here, and its decorator still runs twice. it is not +/// only a side effect that would move: the class *construction* reads what a method +/// decorator wrote, and `ABCMeta` is the case — it computes `__abstractmethods__` from +/// the namespace the body left, so taking `@abstractmethod` out of the twin empties that +/// set on every class whose construction falls back to the interpreted definition. a +/// method's decorator has to be applied where the `def` stands, which means the answer +/// is to carry the *result* across rather than to move the decorator; see the module docs +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InitDecoration<'a> { + /// the name as written, which is the name the twin's source binds it under + pub name: &'a str, + /// outermost first, the order they are written in + pub decorators: &'a [Decorator], +} + impl ModuleIr { + /// every module-level definition init decorates — see [`InitDecoration`] + pub fn decorated_at_init(&self) -> impl Iterator> { + // a closure environment is not in the namespace under any name, so there is no + // entry to apply anything to — and it carries no decorator anyway + let classes = self + .classes + .iter() + .filter(|class| class.exported && !class.decorators.is_empty()) + .map(|class| InitDecoration { + name: class.name.as_str(), + decorators: &class.decorators, + }); + // an unboxed edition is a second `Function` under a mangled name the namespace + // never holds, and it carries the decorators of the function it is an edition + // of. reaching for that name here would fail the import with a `NameError` + let functions = self + .functions + .iter() + .filter(|function| function.exported && !function.decorators.is_empty()) + .map(|function| InitDecoration { + name: function.name.as_str(), + decorators: &function.decorators, + }); + classes.chain(functions) + } + /// every compiled function, methods included /// /// a pass that iterates `functions` alone silently skips every method, which @@ -634,7 +794,7 @@ impl ModuleIr { ) } - pub fn new(name: impl Into) -> Self { + pub fn new(name: impl Into) -> Self { Self { name: name.into(), functions: Vec::new(), @@ -650,8 +810,116 @@ impl ModuleIr { /// the C identifier for the module's init function, which cpython looks up by /// name when loading the extension pub fn init_symbol(&self) -> String { - let last = self.name.rsplit('.').next().unwrap_or(&self.name); - format!("PyInit_{}", mangle(last)) + format!("PyInit_{}", mangle(self.name.last_component())) + } +} + +/// a module's name, in the forms a build needs +/// +/// they are different things, and conflating them is what made a class in +/// `tkinter/m.py` answer `__module__ == "m"`. only the dotted name is stored, so +/// they cannot drift apart: the shorter ones are derived from it on demand +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct ModuleName { + dotted: String, + /// whether the name belongs to a package's `__init__` rather than to a plain + /// module + /// + /// the dotted name cannot say, and the difference decides which file the + /// artefact is written to: `a.b` is `a/b.py` for one and `a/b/__init__.py` + /// for the other + is_package: bool, +} + +impl ModuleName { + /// a plain module — one written in a file named after its own last component + pub fn new(dotted: impl Into) -> Self { + Self { + dotted: dotted.into(), + is_package: false, + } + } + + /// a package, written in the `__init__` inside the directory the name spells + /// + /// `dotted` is the package's own name, not its `__init__`'s: `a/b/__init__.py` + /// is the module `a.b`, because that is what python imports it as and what a + /// class written in it reports as its `__module__` + pub fn package(dotted: impl Into) -> Self { + Self { + dotted: dotted.into(), + is_package: true, + } + } + + /// the name python imports the module as, dots and all + /// + /// this is what a type's `tp_name` carries: cpython splits `tp_name` at its + /// last dot to answer `__module__` and `__name__`, so a class in the module + /// `tkinter.m` needs the whole of it to say where it came from. + /// + /// it is also what every emitted C identifier is prefixed with, once the + /// mangling has turned the dots into underscores — two modules of the same + /// name in different packages then get different symbols + pub fn dotted(&self) -> &str { + &self.dotted + } + + /// the last dotted component — the module's own name inside its package + /// + /// this is cpython's rule rather than ours: cpython derives the init function + /// it looks up from the last component of the *spec* name, so an extension + /// imported as `tkinter.m` has to export `PyInit_m` however deep the package + /// goes, and a package imported as `tkinter.sub` exports `PyInit_sub` even + /// though its file is called `__init__` + /// + /// it is emphatically **not** the file name — see [`Self::relative_path`] + pub fn last_component(&self) -> &str { + self.dotted.rsplit('.').next().unwrap_or(&self.dotted) + } + + /// where the module's file belongs inside an output tree, given the suffix the + /// file carries (`".c"`, or the interpreter's extension suffix) + /// + /// the tree has to mirror the module tree, because that is the only layout + /// python will import these back under: `a.b.c` is only reachable as + /// `a/b/c`. a flat directory can offer nothing but the last component, + /// which is both unimportable as a package member and a collision — two + /// members of one package ending in the same component overwrite each other. + /// + /// a package is the case the dotted name alone gets wrong: `a.b` as a package + /// is the *directory* `a/b`, and its file is the `__init__` inside it + pub fn relative_path(&self, suffix: &str) -> PathBuf { + let mut directories: Vec<&str> = self + .dotted + .split('.') + // an empty component would contribute nothing to the path but would + // shift which component names the file, so it is dropped rather than + // joined + .filter(|component| !component.is_empty()) + .collect(); + // a package's last component names the directory holding its `__init__`; a + // plain module's names the file itself + let stem = if self.is_package { + "__init__" + } else { + directories.pop().unwrap_or_default() + }; + let mut path: PathBuf = directories.into_iter().collect(); + path.push(format!("{stem}{suffix}")); + path + } +} + +impl From<&str> for ModuleName { + fn from(dotted: &str) -> Self { + Self::new(dotted) + } +} + +impl From for ModuleName { + fn from(dotted: String) -> Self { + Self::new(dotted) } } @@ -664,6 +932,8 @@ fn mangle(name: &str) -> String { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; use crate::ops::BlockId; @@ -701,9 +971,30 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: Binding::Instance, } } + /// an unboxed edition is a second `Function` under a mangled name, carrying the + /// decorators of the function it is an edition of. the namespace never binds that + /// name, so a decorator applied to it at init would raise `NameError` and take the + /// whole import with it + #[test] + fn a_definition_the_namespace_does_not_bind_is_not_decorated_at_init() { + let mut module = ModuleIr::new("m"); + let mut exported = function(); + exported.decorators = vec![Decorator::name("mark")]; + let mut edition = exported.clone(); + edition.name = "add$arr0l".to_string(); + edition.exported = false; + module.functions = vec![exported, edition]; + let named: Vec<&str> = module + .decorated_at_init() + .map(|decoration| decoration.name) + .collect(); + assert_eq!(named, ["add"]); + } + #[test] fn symbols_are_c_identifiers() { let f = function(); @@ -713,12 +1004,73 @@ mod tests { #[test] fn the_init_symbol_uses_only_the_last_component() { - // cpython looks up PyInit_ in the loaded object + // cpython looks up PyInit_ let module = ModuleIr::new("pkg.sub.mod"); assert_eq!(module.init_symbol(), "PyInit_mod"); assert_eq!(ModuleIr::new("mod").init_symbol(), "PyInit_mod"); } + /// a package's file is called `__init__`, but cpython still derives the init + /// symbol from the last component of the name it is imported under — so the + /// symbol and the file name disagree, and only for a package + #[test] + fn a_packages_init_symbol_is_named_after_the_package_not_its_file() { + let module = ModuleIr::new(ModuleName::package("pkg.sub")); + assert_eq!(module.init_symbol(), "PyInit_sub"); + assert_eq!( + module.name.relative_path(".so"), + Path::new("pkg/sub/__init__.so") + ); + } + + /// the two halves a module name has to keep apart: what python imports the + /// module as, and the shorter name cpython's loader uses + #[test] + fn a_module_name_keeps_its_dotted_form_and_its_last_component() { + let name = ModuleName::new("pkg.sub.mod"); + assert_eq!(name.dotted(), "pkg.sub.mod"); + assert_eq!(name.last_component(), "mod"); + + // a top-level module is the same string twice, which is why the two were + // conflated for so long + let alone = ModuleName::new("mod"); + assert_eq!(alone.dotted(), "mod"); + assert_eq!(alone.last_component(), "mod"); + } + + /// the artefact tree is the module tree: a name is only importable back from + /// the path its dots spell out + #[test] + fn a_module_name_spells_out_the_path_its_artefact_is_written_to() { + assert_eq!( + ModuleName::new("pkg.sub.mod").relative_path(".c"), + Path::new("pkg/sub/mod.c") + ); + assert_eq!( + ModuleName::package("pkg.sub").relative_path(".c"), + Path::new("pkg/sub/__init__.c") + ); + assert_eq!( + ModuleName::new("mod").relative_path(".c"), + Path::new("mod.c") + ); + assert_eq!( + ModuleName::package("pkg").relative_path(".cpython-313-darwin.so"), + Path::new("pkg/__init__.cpython-313-darwin.so") + ); + } + + /// two members of one package can share a last component, and a flat output + /// directory would give them the same file — this is the collision the tree + /// layout removes + #[test] + fn two_package_members_sharing_a_last_component_take_different_paths() { + assert_ne!( + ModuleName::new("pkg.dup").relative_path(".so"), + ModuleName::new("pkg.sub.dup").relative_path(".so") + ); + } + #[test] fn params_are_the_leading_registers() { let f = function(); diff --git a/crates/by_ir/src/lib.rs b/crates/by_ir/src/lib.rs index f36d9a172d..023f9dfeeb 100644 --- a/crates/by_ir/src/lib.rs +++ b/crates/by_ir/src/lib.rs @@ -17,8 +17,8 @@ pub mod verify; pub use builder::FunctionBuilder; pub use function::{ - BasicBlock, CallConvention, ClassIr, Declined, FieldDecl, Function, GradualUse, ModuleIr, - RegisterDecl, + BasicBlock, CallConvention, ClassIr, Declined, Decorator, FieldDecl, Function, GradualUse, + ModuleIr, ModuleName, RegisterDecl, }; pub use ops::{BinOp, BlockId, CmpOp, Op, RegisterId, StandardError, Terminator, UnaryOp, Value}; pub use print::{print_function, print_module}; diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index a44d51a019..3678d1588e 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -650,6 +650,23 @@ pub enum Op { /// resolved on every read for the same reason a call is — a module global may /// be rebound, and python would see it LoadGlobal { dest: RegisterId, name: String }, + /// bind a name in the module namespace: what an assignment under a `global` + /// declaration does + /// + /// the other half of [`Self::LoadGlobal`], and it has to reach the same place. a + /// register write is private to the frame, where python's binding is the module's + /// — visible at once to every other reader, the interpreted twin included + StoreGlobal { + /// the status of the store, zero on success + dest: RegisterId, + name: String, + value: Value, + }, + /// unbind a name in the module namespace: `del x` under a `global x` + /// + /// a name that is not bound is a `NameError`, which is not what deleting from a + /// dict raises — see `By_DeleteGlobal` + DeleteGlobal { dest: RegisterId, name: String }, /// the type object of a class this module emits, by identity rather than by name /// /// a class decorator replaces the *namespace* entry, and a module may rebind the @@ -864,6 +881,8 @@ impl Op { | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } | Self::LoadGlobal { dest, .. } + | Self::StoreGlobal { dest, .. } + | Self::DeleteGlobal { dest, .. } | Self::LoadClass { dest, .. } | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } @@ -956,6 +975,8 @@ impl Op { | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } | Self::LoadGlobal { dest, .. } + | Self::StoreGlobal { dest, .. } + | Self::DeleteGlobal { dest, .. } | Self::LoadClass { dest, .. } | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } @@ -1099,6 +1120,7 @@ impl Op { Self::SetAttr { receiver, value, .. } => vec![receiver, value], + Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } | Self::BuildList { items, .. } | Self::BuildSet { items, .. } @@ -1126,6 +1148,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::DeleteGlobal { .. } | Self::LoadClass { .. } | Self::ImportModule { .. } => Vec::new(), Self::Enter { manager, .. } => vec![manager], @@ -1266,6 +1289,7 @@ impl Op { Self::SetAttr { receiver, value, .. } => vec![receiver, value], + Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } | Self::BuildList { items, .. } | Self::BuildSet { items, .. } @@ -1293,6 +1317,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::DeleteGlobal { .. } | Self::LoadClass { .. } | Self::ImportModule { .. } => Vec::new(), Self::Enter { manager, .. } => vec![manager], diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 517a21b1cc..0b7bd4a7af 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -11,7 +11,7 @@ use crate::ops::{Mutation, Op, RegisterId, Terminator, UnaryOp, Value}; /// render a whole module pub fn print_module(module: &ModuleIr) -> String { - let mut out = format!("module {}\n", module.name); + let mut out = format!("module {}\n", module.name.dotted()); for function in &module.functions { out.push('\n'); out.push_str(&print_function(function)); @@ -434,6 +434,14 @@ fn print_op(function: &Function, op: &Op) -> String { Op::LoadGlobal { dest, name: global } => { format!("{} = global {global}", name(*dest)) } + Op::StoreGlobal { + dest, + name: global, + value: v, + } => format!("{} = global {global} <- {}", name(*dest), value(v)), + Op::DeleteGlobal { dest, name: global } => { + format!("{} = del global {global}", name(*dest)) + } Op::LoadClass { dest, class } => { format!("{} = class {class}", name(*dest)) } @@ -699,6 +707,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, } } @@ -724,7 +733,7 @@ b2: #[test] fn a_module_lists_declines_after_its_functions() { let module = ModuleIr { - name: "app".to_string(), + name: crate::ModuleName::new("app"), functions: vec![sample()], declined: vec![Declined { range: None, @@ -764,6 +773,7 @@ b2: decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; assert!(print_function(&function).contains("return 1.0")); } diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index bea84b404e..c329901ddd 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -933,6 +933,14 @@ impl Verifier<'_> { Op::LoadGlobal { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a global read"); } + Op::StoreGlobal { dest, value, .. } => { + // the namespace holds objects, so a write to one has to arrive boxed + self.expect(block, value, &RType::OBJECT, "a global write"); + self.expect_dest(block, *dest, &RType::BIT, "a global write"); + } + Op::DeleteGlobal { dest, .. } => { + self.expect_dest(block, *dest, &RType::BIT, "a global delete"); + } Op::LoadClass { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a class read"); } @@ -1453,6 +1461,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, } } @@ -1638,6 +1647,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!( @@ -1684,6 +1694,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; assert_eq!(verify(&f), Ok(())); } @@ -1731,6 +1742,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!( @@ -1829,6 +1841,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!(errors.iter().any(|e| e.message.contains("past the end"))); @@ -1840,7 +1853,7 @@ mod tests { broken.name = "broken".to_string(); broken.ret = RType::FLOAT; let module = ModuleIr { - name: "m".to_string(), + name: crate::ModuleName::new("m"), functions: vec![add(), broken], declined: Vec::new(), classes: Vec::new(), @@ -1875,7 +1888,7 @@ mod tests { }, ); let module = ModuleIr { - name: "m".to_string(), + name: crate::ModuleName::new("m"), functions: vec![add(), caller], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_irbuild/Cargo.toml b/crates/by_irbuild/Cargo.toml index f464b30b0c..7ecf849cde 100644 --- a/crates/by_irbuild/Cargo.toml +++ b/crates/by_irbuild/Cargo.toml @@ -14,6 +14,7 @@ license = "MIT OR Apache-2.0" by_ir = { workspace = true } ruff_db = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } thin-vec = { workspace = true } ruff_text_size = { workspace = true } diff --git a/crates/by_irbuild/src/closures.rs b/crates/by_irbuild/src/closures.rs index ab30c46432..34da387f51 100644 --- a/crates/by_irbuild/src/closures.rs +++ b/crates/by_irbuild/src/closures.rs @@ -107,18 +107,6 @@ pub(crate) fn nested_functions( for (def, lambda) in definitions { let def = &def; - // a decorator on a nested function wraps the closure where the `def` - // stands, so it is resolved the way the module-level ones are: a plain - // name, looked up as `LOAD_GLOBAL` would - if def - .decorator_list - .iter() - .any(|decorator| !matches!(&decorator.expression, ast::Expr::Name(_))) - { - return Err(Decline::new( - "only a plain-name decorator on a nested function is lowered yet", - )); - } let own = own_names(def); let mut captures: Vec = Vec::new(); let mut seen: HashSet<&str> = HashSet::new(); @@ -201,6 +189,10 @@ fn own_names(def: &ast::StmtFunctionDef) -> HashSet<&str> { out.insert(kwarg.name.as_str()); } out.extend(written_names(&def.body)); + // a name declared `global` here resolves in the module namespace whether this body + // writes it or only reads it, so an enclosing local of the same name must never be + // captured for it. counting it as this function's own is what says so + out.extend(global_names(def)); // a `nonlocal` name is explicitly *not* the nested function's own for name in nonlocal_names(def) { out.remove(name); @@ -222,6 +214,18 @@ fn nonlocal_names(def: &ast::StmtFunctionDef) -> Vec<&str> { .collect() } +/// the names a function declares `global` +fn global_names(def: &ast::StmtFunctionDef) -> Vec<&str> { + crate::walk(&def.body) + .into_iter() + .filter_map(|stmt| match stmt { + Stmt::Global(node) => Some(node.names.iter().map(ruff_python_ast::Identifier::as_str)), + _ => None, + }) + .flatten() + .collect() +} + /// every name a body assigns to pub(crate) fn written_names(body: &[Stmt]) -> Vec<&str> { let mut out = Vec::new(); @@ -270,6 +274,12 @@ fn read_names(body: &[Stmt]) -> Vec<&str> { // and a lambda's body, which is an expression the walk above does reach — but a // nested `def`'s body is a statement list the walk deliberately stops at if let Stmt::FunctionDef(nested) = stmt { + // a decorator belongs to the frame the `def` stands in, not to the function + // it decorates: it is evaluated there, so the names in it are read here and + // filtered by nothing the nested function binds + for decorator in &nested.decorator_list { + collect_reads(&decorator.expression, &mut out); + } let own = own_names(nested); out.extend( read_names(&nested.body) diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index d727ddcb61..7eb9bae8ef 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -52,8 +52,8 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use by_ir::builder::FunctionBuilder; use by_ir::function::{ - CallConvention, ClassBase, ClassKeyword, Declined, Function, GradualUse, KeywordValue, - ModuleIr, qualify, + Binding, CallConvention, ClassBase, ClassKeyword, Declined, Decorator, Function, GradualUse, + KeywordValue, ModuleIr, ModuleName, qualify, }; use by_ir::ops::{ BinOp, BlockId, CmpOp, Conversion, Mutation, Op, RegisterId, StandardError, Terminator, @@ -70,12 +70,16 @@ use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::{HasType, SemanticModel}; /// lower every module-level function ty can represent natively +/// +/// `module_name` is the **dotted** name python imports the module as, because +/// that is what a class in it has to answer for `__module__` — see +/// [`by_ir::ModuleName`] pub fn build_module( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, model: &SemanticModel<'_>, suite: &[Stmt], - module_name: &str, + module_name: impl Into, unique_loop_bindings: bool, ) -> ModuleIr { let mut module = ModuleIr::new(module_name); @@ -89,6 +93,51 @@ pub fn build_module( _ => None, }) .collect(); + // …and only when the *name* still holds that function. a decorator replaces what + // the module namespace binds with whatever it returned, which may be another + // function, a class, a descriptor, or no callable at all — so a call through the + // name has to go out through the namespace and find it. classes are in here for + // the same reason: a construction is written against the name, not against the + // type this module emitted under it. + // + // this is a separate set from `native_callees` because that one answers "does this + // module declare the name at all", which is what says whether `len`, `range` and + // `super` are still the builtins — and a decorator does not change that + // + // a *modifier* is not a decorator and rebinds nothing, so the question is which + // decorators survive translation rather than which definitions carry one. + // + // a name a frame declares `global` is in here for the third form of the same + // reason: the frame is going to rebind it, and the definition this module emitted + // under that name is then not what the name holds. reaching it directly answered + // with the old function and refused the new class outright + let rebinds = |decorators: &[ast::Decorator], class: bool| { + decorators.iter().any(|decorator| { + let role = if class { + class_modifier(db, model, decorator) + } else { + function_modifier(db, model, decorator) + }; + // one that declines takes its definition with it, so either answer is safe + !matches!(role, Ok(Modifier::Erased | Modifier::DataClass)) + }) + }; + let decorated: HashSet = suite + .iter() + .filter_map(|stmt| match stmt { + Stmt::FunctionDef(function) if rebinds(&function.decorator_list, false) => { + Some(function.name.to_string()) + } + Stmt::ClassDef(class) if rebinds(&class.decorator_list, true) => { + Some(class.name.to_string()) + } + _ => None, + }) + .chain(declared_global_anywhere(suite)) + .collect(); + + // every name this module reads anywhere — see `names_read` + let read = names_read(suite); // pass one: which classes get an emitted layout. a body cannot be lowered // until this is known, because whether `self.x` is a field read or a @@ -158,14 +207,13 @@ pub fn build_module( .collect(); let mut mutable: HashSet<&str> = HashSet::new(); for class in &declared { - let decorated = class.decorator_list.iter().any(|decorator| { - !matches!(&decorator.expression, Expr::Name(name) - if matches!(name.id.as_str(), "data_class" | "frozen_data_class")) - }); - if decorated { + if decorated.contains(class.name.as_str()) { mutable.insert(class.name.as_str()); } - if let Some(base) = base_class(db, env, model, class, &layouts).ok().flatten() { + if let Some(base) = base_class(db, env, model, suite, class, &layouts) + .ok() + .flatten() + { mutable.insert(class.name.as_str()); // a base of ours is made mutable too, whether it stands alone or beside a // name from outside: this class may override a method of it, and a direct @@ -327,7 +375,7 @@ pub fn build_module( }) .filter_map(|class| { // the map is the *layout* chain, which only an in-module base extends - base_class(db, env, model, class, &layouts) + base_class(db, env, model, suite, class, &layouts) .ok() .flatten() .and_then(|base| base.in_module().map(str::to_owned)) @@ -343,6 +391,8 @@ pub fn build_module( env, model, native_callees: &native_callees, + decorated: &decorated, + read: &read, suite, layouts: &layouts, methods: &methods, @@ -376,7 +426,10 @@ pub fn build_module( module .promoted .extend(promoted_places(db, env, model, function, &layouts)); - match lower_function(unit, function).and_then(verified) { + match defined_once(suite, function) + .and_then(|()| lower_function(unit, function)) + .and_then(verified) + { Ok((lowered, environments)) => { module.functions.push(lowered); // an environment is a real emitted class, just not a named one @@ -413,7 +466,8 @@ pub fn build_module( // every class the module writes, and the names its header extends. syntactic // on purpose: what the interpreted definition builds on is whatever the name // resolves to in the module namespace when the `class` statement runs, which - // is this module's compiled type wherever it emitted one + // is this module's compiled type wherever it emitted one. a name the module + // aliases stands for the class it was bound to, the same as in a base let extends: Vec<(String, Vec)> = suite .iter() .filter_map(|stmt| match stmt { @@ -423,7 +477,15 @@ pub fn build_module( .bases() .iter() .filter_map(|base| match base { - Expr::Name(name) => Some(name.id.to_string()), + Expr::Name(name) => { + // a name this cannot settle stands for itself here, which is + // what the interpreted definition looked up + Some( + base_stands_for(suite, name.id.as_str()) + .unwrap_or(name.id.as_str()) + .to_string(), + ) + } _ => None, }) .collect(), @@ -431,12 +493,24 @@ pub fn build_module( _ => None, }) .collect(); - prune_unbuildable(&mut module, &ranges, &extends, &shadowed_definitions(suite)); + prune_unbuildable( + &mut module, + &ranges, + &extends, + &disturbed_definitions(suite), + ); module } -/// every module-level `def` or `class` whose name the module body binds again -/// afterwards +/// what the module body does to its own definitions after making them +struct Disturbed { + /// every `def` or `class` whose name the body binds again + rebound: HashSet, + /// every class the body writes a dunder attribute onto, and one such name + dunder: HashMap, +} + +/// every module-level `def` or `class` the module body disturbs after defining it /// /// module init installs the native definition into the namespace over whatever the /// fallback source left there, which is the definition it replaces only while nothing @@ -450,7 +524,13 @@ pub fn build_module( /// a binding *before* the definition is the ordinary forward declaration — /// `Enum = Flag = ReprEnum = None` ahead of the classes themselves — which the /// definition then overwrites, so the two are compared by position -fn shadowed_definitions(suite: &[Stmt]) -> HashSet { +/// +/// a dunder written onto a *class* is the other half, and it is not about the name at +/// all: what an emitted type takes from its twin is the twin's own dict minus the +/// dunders, because a dunder is what a type slot answers and a second answer in the dict +/// would disagree with it. so a dunder the body wrote there has nowhere to land, and +/// where it is compared by position the ordinary forward declaration is +fn disturbed_definitions(suite: &[Stmt]) -> Disturbed { let defined: HashMap<&str, TextSize> = suite .iter() .filter_map(|stmt| match stmt { @@ -459,14 +539,80 @@ fn shadowed_definitions(suite: &[Stmt]) -> HashSet { _ => None, }) .collect(); - let mut bindings = ModuleBindings { found: Vec::new() }; + let mut bindings = ModuleBindings { + found: Vec::new(), + unbinds: false, + dunders: Vec::new(), + }; ast::visitor::walk_body(&mut bindings, suite); - bindings - .found + let classes: HashSet<&str> = suite + .iter() + .filter_map(|stmt| match stmt { + Stmt::ClassDef(class) => Some(class.name.as_str()), + _ => None, + }) + .collect(); + let dunder = bindings + .dunders .into_iter() - .filter(|(name, at)| defined.get(name).is_some_and(|defined| at > defined)) - .map(|(name, _)| name.to_string()) - .collect() + .filter(|(owner, _)| classes.contains(owner)) + .map(|(owner, attribute)| (owner.to_string(), attribute.to_string())) + .collect(); + // a name taken back out of the namespace object is a `del` whose target this cannot + // read — `ast` pops five of its own classes out through a comprehension — so every + // definition is treated as one it could have been + if bindings.unbinds { + return Disturbed { + rebound: defined.into_keys().map(str::to_string).collect(), + dunder, + }; + } + Disturbed { + rebound: bindings + .found + .into_iter() + .filter(|(name, at)| defined.get(name).is_some_and(|defined| at > defined)) + .map(|(name, _)| name.to_string()) + .collect(), + dunder, + } +} + +/// whether a name is one python spells with two underscores at each end +/// +/// the same test `By_IsDunder` makes, which is what decides at runtime whether an +/// attribute is carried from a twin +fn is_a_dunder(name: &str) -> bool { + name.len() > 4 && name.starts_with("__") && name.ends_with("__") +} + +/// whether an expression is the module namespace itself +/// +/// only the call, not a name it was stored in first: what a namespace held somewhere else +/// can be made to do is a wider question than this one, and the answer to it is not +/// syntactic +fn module_namespace(expr: &Expr) -> bool { + matches!(expr, Expr::Call(call) + if call.arguments.is_empty() + && matches!(call.func.as_ref(), Expr::Name(name) if name.id.as_str() == "globals")) +} + +/// whether an expression takes a binding back out of the module namespace +fn unbinds_through_the_namespace(expr: &Expr) -> bool { + match expr { + // `del globals()[name]`, which the store/delete context is the whole of + Expr::Subscript(subscript) => { + subscript.ctx == ExprContext::Del && module_namespace(&subscript.value) + } + Expr::Call(call) => match call.func.as_ref() { + Expr::Attribute(attribute) => { + module_namespace(&attribute.value) + && matches!(attribute.attr.as_str(), "pop" | "popitem" | "clear") + } + _ => false, + }, + _ => false, + } } /// every name the module body binds, and where @@ -476,6 +622,10 @@ fn shadowed_definitions(suite: &[Stmt]) -> HashSet { /// that bind an identifier rather than an expression are the remaining cases struct ModuleBindings<'a> { found: Vec<(&'a str, TextSize)>, + /// whether the body took a name back out of its own namespace, without saying which + unbinds: bool, + /// the dunder attributes the body writes onto a name, as `(owner, attribute)` + dunders: Vec<(&'a str, &'a str)>, } impl<'a> ast::visitor::Visitor<'a> for ModuleBindings<'a> { @@ -495,6 +645,15 @@ impl<'a> ast::visitor::Visitor<'a> for ModuleBindings<'a> { { self.found.push((name.id.as_str(), name.range.start())); } + if let Expr::Attribute(attribute) = expr + && matches!(attribute.ctx, ExprContext::Store | ExprContext::Del) + && let Expr::Name(owner) = attribute.value.as_ref() + && is_a_dunder(attribute.attr.as_str()) + { + self.dunders + .push((owner.id.as_str(), attribute.attr.as_str())); + } + self.unbinds |= unbinds_through_the_namespace(expr); ast::visitor::walk_expr(self, expr); } @@ -544,7 +703,7 @@ fn span(range: ruff_text_size::TextRange) -> (u32, u32) { fn lower_generator( unit: Unit<'_>, function: &ast::StmtFunctionDef, - decorators: Vec, + decorators: Vec, receiver: Option>, captures: Option<&closures::Nested>, ) -> Lowered<(Function, Vec)> { @@ -567,8 +726,12 @@ fn lower_generator( // a field is a *cell* — `object`, with an unset check on every read — unless the // name is definitely assigned, in which case it takes the local's own // representation and the read is an infallible `GetField` - let representations = + // a declared `global` is not one of them: it lives in the module namespace, which + // already outlives every suspension + let declared_global = declared_globals(&function.body); + let mut representations = local_representations(db, env, model, &function.body, layouts, unit.arrays); + representations.retain(|(name, _)| !declared_global.contains(name)); let locals: Vec = representations .iter() .map(|(name, _)| name.clone()) @@ -681,7 +844,7 @@ fn lower_generator_constructor( function: &ast::StmtFunctionDef, class: &str, fields: &[by_ir::function::FieldDecl], - decorators: Vec, + decorators: Vec, receiver: Option>, captured: &[String], ) -> Lowered { @@ -836,7 +999,9 @@ fn lower_resume( model, builder, locals: HashMap::new(), + globals: declared_globals(&function.body), native_callees, + decorated: unit.decorated, layouts, methods, signatures, @@ -1009,21 +1174,34 @@ fn verify_one(function: &mut Function) -> Lowered<()> { /// /// `extends` is every class the module writes and the names its header extends, /// including the ones nothing here will emit — a class left to the interpreted -/// definition still extends what it says it extends. `shadowed` is every name the -/// module body rebinds after defining it, from [`shadowed_definitions`] +/// definition still extends what it says it extends. `disturbed` is what the module body +/// does to its own definitions after making them, from [`disturbed_definitions`] fn prune_unbuildable( module: &mut ModuleIr, ranges: &HashMap, extends: &[(String, Vec)], - shadowed: &HashSet, + disturbed: &Disturbed, ) { // the name is the whole of what module init has to install under, and only an // exported definition is installed at all let rebound = |name: &str, exported: bool| { - (exported && shadowed.contains(name)).then(|| { + (exported && disturbed.rebound.contains(name)).then(|| { format!("`{name}` is rebound at module level, so installing this over it would replace what the rebind produced") }) }; + // and what the body hangs on the definition afterwards has to survive the swap too. + // `ctypes` writes `c_byte.__ctype_le__` there, and the adoption that carries a twin's + // attributes leaves every dunder behind + let hung_on = |name: &str, exported: bool| { + exported + .then(|| disturbed.dunder.get(name)) + .flatten() + .map(|attribute| { + format!( + "the module body writes `{attribute}` onto `{name}`, which the emitted type does not carry" + ) + }) + }; loop { let classes: HashSet = module .classes @@ -1036,6 +1214,7 @@ fn prune_unbuildable( .all_functions() .map(Function::qualified_name) .collect(); + let anchors = storage_anchors(module); let unbuildable = |function: &Function| -> Option { let representations = function @@ -1071,15 +1250,35 @@ fn prune_unbuildable( // a class goes as a unit: the native type object replaces the interpreted // class whole, so keeping it with one method missing would drop that // method from the module's surface - module.classes.retain(|class| { + module.classes.retain_mut(|class| { // a base this module meant to emit and did not is not a base at all, and // building on nothing in its place would quietly drop everything it brought + // + // unless this class brought no storage, and nothing under it did either. what + // stands under the base's name at import is a class either way — the + // interpreted definition, where the base declined — and building on the + // *name* is what every class over a base out of this module already does. + // + // a class with fields has no such answer: its struct begins with the base's, + // at offsets only the emitted base has. neither has one whose *subclass* + // stores something, because rebuilding this class moves the whole chain's + // layout outside the module, and that subclass's fields would go from inside + // an instance to past one — which is a construction it has no answer for + // either, and one that refuses the whole module at import rather than the + // class. `urllib.request` lost every compiled definition it had that way let declined_base = class .base .as_ref() .and_then(ClassBase::in_module) .filter(|base| !classes.contains(*base)) - .map(|base| format!("`{base}` declined, so it is not a base to build on")); + .map(str::to_owned) + .and_then(|base| { + if class.fields.is_empty() && !anchors.contains(&class.name) { + class.base = Some(ClassBase::External(vec![base])); + return None; + } + Some(format!("`{base}` declined, so it is not a base to build on")) + }); // and the other way round. a class this module does not emit is still // built — by the interpreted definition, on whatever its base name // resolves to, which is the type emitted here. that is a subclass an @@ -1095,6 +1294,7 @@ fn prune_unbuildable( }) }; match rebound(&class.name, class.exported) + .or_else(|| hung_on(&class.name, class.exported)) .or(declined_base) .or_else(interpreted_subclass) .or_else(|| class.methods.iter().find_map(&unbuildable)) @@ -1131,6 +1331,34 @@ fn prune_unbuildable( } } +/// every class an emitted one keeps storage inside an instance of +/// +/// a class's struct begins with the fields of every class this module writes above it, so +/// where any of those stops being a layout of ours the storage stops being *inside* the +/// instance and starts sitting past one. these are the classes that cannot be moved +fn storage_anchors(module: &ModuleIr) -> HashSet { + let mut anchors = HashSet::new(); + for class in &module.classes { + if class.fields.is_empty() { + continue; + } + let mut current = class; + // bounded by the class count: a base chain cannot visit one twice without being a + // cycle, and a cycle would otherwise spin here rather than settle + for _ in 0..=module.classes.len() { + let Some(base) = current.base.as_ref().and_then(ClassBase::in_module) else { + break; + }; + anchors.insert(base.to_string()); + match module.classes.iter().find(|other| other.name == base) { + Some(next) => current = next, + None => break, + } + } + } + anchors +} + /// what declined /// the places in `function` a representation was available at but for the promotion /// @@ -1250,25 +1478,8 @@ fn lower_class<'a>( // class form with a field-initializing constructor — which is exactly what a // fixed layout needs. a plain class with bare annotations has no constructor // in the interpreted build either, so compiling one would invent behaviour - let mut is_data = false; - // a real decorator is applied to the *namespace entry* after the type is - // installed, and every construction resolves the class through that namespace — - // so it gets whatever the decorator produced - let mut class_decorators = Vec::new(); - for decorator in &class.decorator_list { - match &decorator.expression { - Expr::Name(name) if matches!(name.id.as_str(), "data_class" | "frozen_data_class") => { - is_data = true; - } - Expr::Name(name) => class_decorators.push(name.id.to_string()), - _ => { - return Err(Decline::new( - "only a plain-name class decorator is lowered yet", - )); - } - } - } - let base = base_class(db, env, model, class, layouts)?; + let (is_data, class_decorators) = class_modifiers(db, model, class)?; + let base = base_class(db, env, model, suite, class, layouts)?; let fields = class_fields(db, env, model, suite, class, layouts)?; let mut lowered = Vec::new(); @@ -1401,6 +1612,23 @@ fn lower_class<'a>( method.name ))); } + // every decorator is resolved out of the module namespace at init, so + // one rooted at a name the *class body* bound is looked up somewhere it + // does not exist. `@fieldnames.setter` is the shape this turns down, and + // it turns down the whole class: two `def`s of one name would otherwise + // become two entries in one method table + if let Some(rooted) = method.decorator_list.iter().find_map(|decorator| { + let path = function_modifier(db, model, decorator).ok()?; + let Modifier::Apply(path) = path else { + return None; + }; + class_body_binds(&class.body, path.root()).then(|| path.root().to_string()) + }) { + return Err(Decline::new(format!( + "`{rooted}` is bound by the class body, and a decorator is resolved out of the module namespace at init" + ))); + } + defined_once(&class.body, method)?; let (method, produced) = lower_method(unit, method, &class.name)?; lowered.push(method); environments.extend(produced); @@ -1433,7 +1661,11 @@ fn lower_class<'a>( "`{clash}` is both a class-level constant and a field" ))); } - + // this class's own decorators are applied at init and taken out of the twin's source, + // so the module body must not reach the class in the window between the twin's + // `class` statement and init — everything it bound in that window keeps the + // definition nothing had decorated yet + decorator_stays_unread(unit.read, class.name.as_str(), &class_decorators)?; Ok(( by_ir::function::ClassIr { resume: None, @@ -1727,6 +1959,526 @@ fn attribute_of(target: &Expr, receiver: &str, owner: Option<&str>) -> Option, + class: &ast::StmtClassDef, +) -> Lowered<(bool, Vec)> { + let mut is_data = false; + let mut applied = Vec::new(); + for decorator in &class.decorator_list { + match class_modifier(db, model, decorator)? { + Modifier::DataClass => is_data = true, + Modifier::Erased => {} + Modifier::Apply(decorator) => applied.push(decorator), + } + } + if !applied.is_empty() + && let Some(unwritten) = published_beyond_the_body(class) + { + return Err(Decline::new(format!( + "an emitted type publishes `{unwritten}` alongside a method this class writes, and a decorator reads the class it is handed" + ))); + } + Ok((is_data, applied)) +} + +/// the dunders an emitted type publishes alongside this one, because they share a slot +/// +/// python reaches these through a *slot* rather than by name, and one slot backs several +/// names: `tp_richcompare` backs all six comparisons, every binary numeric slot backs an +/// operator and its reflection, and `mp_ass_subscript` backs `__setitem__` along with +/// `__delitem__`. the type publishes a wrapper for each name a filled slot backs, so a +/// class that writes one of a group gets the whole group and the rest answer +/// `NotImplemented`. +/// +/// the groups are `COMPARISONS`, `ARITHMETIC`, `POWER` and `slot_companion` in +/// `by_codegen_c`, which sits downstream of this crate and cannot be asked from here +fn shares_a_slot(name: &str) -> &'static [&'static str] { + const GROUPS: &[&[&str]] = &[ + &["__lt__", "__le__", "__eq__", "__ne__", "__gt__", "__ge__"], + &["__add__", "__radd__"], + &["__sub__", "__rsub__"], + &["__mul__", "__rmul__"], + &["__truediv__", "__rtruediv__"], + &["__floordiv__", "__rfloordiv__"], + &["__mod__", "__rmod__"], + &["__divmod__", "__rdivmod__"], + &["__lshift__", "__rlshift__"], + &["__rshift__", "__rrshift__"], + &["__and__", "__rand__"], + &["__xor__", "__rxor__"], + &["__or__", "__ror__"], + &["__matmul__", "__rmatmul__"], + &["__pow__", "__rpow__"], + &["__setitem__", "__delitem__"], + ]; + GROUPS + .iter() + .find(|group| group.contains(&name)) + .copied() + .unwrap_or(&[]) +} + +/// a name the emitted type would publish that the class body never wrote, where the +/// class writes any method at all that shares its slot with one +/// +/// this is what stops a decorator being handed a class the `class` statement did not +/// write. `@functools.total_ordering` is the shape that proves it matters: it fills in +/// the comparisons a class left out, saw `__le__` already published, added nothing — and +/// `a <= b` then raised where the interpreted class answered `True` +fn published_beyond_the_body(class: &ast::StmtClassDef) -> Option<&'static str> { + let written = |name: &str| { + class + .body + .iter() + .any(|statement| matches!(statement, Stmt::FunctionDef(method) if method.name.as_str() == name)) + }; + class.body.iter().find_map(|statement| { + let Stmt::FunctionDef(method) = statement else { + return None; + }; + shares_a_slot(method.name.as_str()) + .iter() + .copied() + .find(|name| !written(name)) + }) +} + +/// what a modifier keyword means to the native build +enum Modifier { + /// the `data class` marker, which is a layout rather than a decorator + DataClass, + /// no runtime effect at all: the transpiler erases it, so the interpreted twin + /// has nothing there either + Erased, + /// a real python decorator, applied to the finished definition at module init — + /// which is what the transpiler emits in a modifier's place + Apply(Decorator), +} + +/// the decorator expression this is, or why it is one the native build cannot evaluate +/// +/// python evaluates a decorator where the definition stands. a module-level definition +/// stands in the interpreted twin's body, which has already run by the time module init +/// installs the native one — so init is the only moment left, and the expression has to +/// be one that means the same thing there. a chain of attribute reads off a name does; a +/// call does not, because calling it at init calls it a *second* time and at the end of +/// the module rather than where it was written +fn decorator_path(expression: &Expr) -> Lowered { + let mut attributes = Vec::new(); + let mut cursor = expression; + loop { + match cursor { + Expr::Name(name) => { + attributes.reverse(); + return Ok(Decorator::Path { + root: name.id.to_string(), + attributes, + }); + } + Expr::Attribute(attribute) => { + attributes.push(attribute.attr.to_string()); + cursor = &attribute.value; + } + Expr::Call(_) => { + return Err(Decline::new( + "a decorator that is a call is evaluated where the definition stands, and module-level code is not compiled — calling it at init would run it a second time", + )); + } + _ => { + return Err(Decline::new( + "only a name, or a chain of attributes read off one, is lowered as a decorator", + )); + } + } + } +} + +/// whether a module-level definition can have its decorators moved to module init +/// +/// python runs a decorator where the definition stands, and the twin's body is what +/// stands there — so init running it again is one evaluation too many, and the decorator +/// comes out of the twin's source to make it one. that leaves a window, from the twin's +/// `def` to the end of module init, in which the name holds a definition nothing has +/// decorated yet. it is invisible unless something reads the name, and the module's own +/// body is the only thing that can: everything else runs after init. +/// +/// so this is what decides whether the move is safe, and a definition it turns down +/// declines rather than being compiled and decorated twice +fn decorator_stays_unread( + read: &BTreeSet<&str>, + name: &str, + decorators: &[Decorator], +) -> Lowered<()> { + if decorators.is_empty() || !read.contains(name) { + return Ok(()); + } + Err(Decline::new(format!( + "this module reads `{name}`, and its decorator cannot run where the definition stands and again over the compiled one" + ))) +} + +/// every name this module's own body can read before module init has finished +/// +/// a decorator module init applies is taken out of the source the twin runs — see +/// [`without_init_decorators`] — so from the twin's `def` until init reaches it, the +/// definition stands in the namespace undecorated. anything that reads the name in that +/// window keeps what it read: `TABLE = f()` in the module body straightforwardly, and +/// `def g(): return f()` called from that body just the same, because `g` reads the +/// global when it runs and not when it was written. +/// +/// a load inside a `def` is a different matter: it happens when that definition *runs*, +/// and everything that runs after import sees the decorated name. so the body reaches one +/// only by naming the definition it is in — which is itself a load the body makes, and is +/// followed from there. a module that defines helpers and calls none of them at import +/// reads nothing at all, which is the common shape and the one that keeps compiling. +/// +/// an annotation is a read only where python evaluates one. under +/// `from __future__ import annotations` it never does — `def f(x: Held)` stores the +/// *string* `"Held"` — so a module written that way names a class in a signature without +/// ever holding what the name meant at that moment +fn names_read(suite: &[Stmt]) -> BTreeSet<&str> { + /// loads the module body makes as it runs, and the loads each definition holds + /// behind its own name + #[derive(Default)] + struct Reads<'a> { + now: BTreeSet<&'a str>, + held: HashMap<&'a str, BTreeSet<&'a str>>, + } + /// a walk that files every load under one heading + struct Into<'a, 'r>(&'r mut BTreeSet<&'a str>); + impl<'a> ruff_python_ast::visitor::Visitor<'a> for Into<'a, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + { + self.0.insert(name.id.as_str()); + } + ruff_python_ast::visitor::walk_expr(self, expr); + } + } + fn statements<'a>(into: &mut BTreeSet<&'a str>, body: &'a [Stmt]) { + for statement in body { + ruff_python_ast::visitor::walk_stmt(&mut Into(into), statement); + } + } + /// everything a `def` evaluates where it stands: its decorators, its defaults, and + /// its annotations where this module evaluates those. only the body waits to be called + fn header<'a>( + into: &mut BTreeSet<&'a str>, + function: &'a ast::StmtFunctionDef, + evaluated: bool, + ) { + let mut visit = Into(into); + for decorator in &function.decorator_list { + ruff_python_ast::visitor::walk_expr(&mut visit, &decorator.expression); + } + for parameter in &function.parameters { + if let Some(annotation) = parameter.annotation() + && evaluated + { + ruff_python_ast::visitor::walk_expr(&mut visit, annotation); + } + if let Some(default) = parameter.default() { + ruff_python_ast::visitor::walk_expr(&mut visit, default); + } + } + if let Some(returns) = &function.returns + && evaluated + { + ruff_python_ast::visitor::walk_expr(&mut visit, returns); + } + } + + let evaluated = annotations_are_evaluated(suite); + let mut reads = Reads::default(); + for statement in suite { + match statement { + Stmt::FunctionDef(function) => { + header(&mut reads.now, function, evaluated); + let held = reads.held.entry(function.name.as_str()).or_default(); + statements(held, &function.body); + } + // a class body runs with the module, so what it reads the module reads. its + // methods' bodies do not, and they wait behind the class's own name + Stmt::ClassDef(class) => { + let mut visit = Into(&mut reads.now); + for decorator in &class.decorator_list { + ruff_python_ast::visitor::walk_expr(&mut visit, &decorator.expression); + } + if let Some(arguments) = &class.arguments { + ruff_python_ast::visitor::walk_arguments(&mut visit, arguments); + } + for member in &class.body { + match member { + Stmt::FunctionDef(method) => { + header(&mut reads.now, method, evaluated); + let held = reads.held.entry(class.name.as_str()).or_default(); + statements(held, &method.body); + } + other => statements(&mut reads.now, std::slice::from_ref(other)), + } + } + } + other => statements(&mut reads.now, std::slice::from_ref(other)), + } + } + + // naming a definition is enough to have reached what it holds: the body may call it, + // hand it to something that calls it, or store it where a later statement will + let mut read = reads.now; + let mut pending: Vec<&str> = read.iter().copied().collect(); + while let Some(name) = pending.pop() { + let Some(held) = reads.held.get(name) else { + continue; + }; + for inner in held { + if read.insert(inner) { + pending.push(inner); + } + } + } + read +} + +/// the interpreted twin's source with every decorator module init re-applies blanked out +/// +/// a decorator is evaluated once in python, where the definition stands. the twin's `def` +/// is what stands there, and module init evaluates the same decorator a *second* time over +/// the native definition that replaces the twin's — so `@register` puts two entries in its +/// registry and `@count_them` counts one function twice. the binding the namespace ends up +/// with is right either way, which is what makes this a silent one. +/// +/// only the decorators [`ModuleIr::decorated_at_init`] names are removed, and each is +/// matched by the path it was written as, so a decorator init does *not* re-apply — the +/// `@dataclass` a `data class` becomes, the `@staticmethod` the method table honours +/// instead, a *method's* which init applies to the finished type — is left where it is +/// and still runs once. +/// +/// they are blanked rather than cut, because a traceback through the twin quotes its +/// source by line: taking the line out would move every definition below it. +/// +/// this is deliberately keyed off the *twin's* text rather than the original source's, +/// because the twin is what runs. for a `.by` module the two are not the same file — the +/// twin is the transpiler's output, where a modifier has already become the decorator it +/// stands for +pub fn without_init_decorators(source: &str, module: &ModuleIr) -> Result { + let mut wanted: HashMap<&str, Vec<&Decorator>> = HashMap::new(); + for decoration in module.decorated_at_init() { + wanted + .entry(decoration.name) + .or_default() + .extend(decoration.decorators); + } + if wanted.is_empty() { + return Ok(source.to_string()); + } + let parsed = ruff_python_parser::parse_module(source) + .map_err(|error| format!("the interpreted fallback does not parse: {error}"))?; + + let mut blank: Vec<(usize, usize)> = Vec::new(); + let mut mark = |paths: Option<&Vec<&Decorator>>, written: &[ast::Decorator]| { + let Some(paths) = paths else { + return; + }; + let mut taken = vec![false; written.len()]; + for path in paths { + // the first unclaimed one that was written as this path. a definition may + // carry the same decorator twice, and then init re-applies it twice too — + // so each application claims one occurrence rather than all of them + let found = written.iter().enumerate().position(|(index, decorator)| { + !taken[index] + && decorator_path(&decorator.expression).is_ok_and(|found| found == **path) + }); + if let Some(index) = found { + taken[index] = true; + let range = written[index].range(); + blank.push((range.start().to_usize(), range.end().to_usize())); + } + } + }; + // only a module-level definition is ever named here, so descending further could + // only blank a decorator on an unrelated definition that happens to share a name + for statement in parsed.suite() { + match statement { + Stmt::FunctionDef(function) => { + mark(wanted.get(function.name.as_str()), &function.decorator_list); + } + Stmt::ClassDef(class) => { + mark(wanted.get(class.name.as_str()), &class.decorator_list); + } + _ => {} + } + } + + let mut out = source.as_bytes().to_vec(); + for (start, end) in blank { + for byte in &mut out[start..end] { + // a decorator may be written over several lines, and the line breaks inside + // it are what keep everything below on the line it was on + if *byte != b'\n' && *byte != b'\r' { + *byte = b' '; + } + } + } + String::from_utf8(out) + .map_err(|error| format!("blanking a decorator split a character: {error}")) +} + +/// whether this definition is the only one of its name in the scope it stands in +/// +/// two `def`s of one name bind whichever one *ran*, so a direct call cannot know which +/// function it is calling — and they mangle to one C symbol besides, which makes the +/// whole module fail to build rather than answer wrongly. `closures::plan` asks this of +/// every nested scope; the module scope had nobody asking, and +/// `importlib/resources/_common.py` has three module-level `def _`s +fn defined_once(scope: &[Stmt], function: &ast::StmtFunctionDef) -> Lowered<()> { + let named = scope + .iter() + .filter( + |statement| matches!(statement, Stmt::FunctionDef(other) if other.name.as_str() == function.name.as_str()), + ) + .count(); + if named > 1 { + return Err(Decline::new(format!( + "`{}` is defined more than once in this scope, so a call to it has no single target", + function.name + ))); + } + Ok(()) +} + +/// whether a class body binds this name +/// +/// a decorator is resolved out of the *module* namespace at module init, and a class +/// body is not that namespace. `@fieldnames.setter` reads the property the body bound +/// two statements up, which is nowhere init can look — so the method keeps its +/// interpreted definition, and with it the whole class +fn class_body_binds(body: &[Stmt], name: &str) -> bool { + body.iter().any(|statement| match statement { + Stmt::FunctionDef(node) => node.name.as_str() == name, + Stmt::ClassDef(node) => node.name.as_str() == name, + Stmt::Assign(node) => node.targets.iter().any(|target| binds_name(target, name)), + Stmt::AnnAssign(node) => binds_name(&node.target, name), + Stmt::AugAssign(node) => binds_name(&node.target, name), + _ => false, + }) +} + +/// whether this module evaluates the annotations it writes +/// +/// `from __future__ import annotations` makes every one of them a string that nothing +/// evaluates until something asks — so naming a class in a signature is not a *read* of +/// it, and a module written that way keeps compiling classes a module without it would +/// have to turn down +fn annotations_are_evaluated(suite: &[Stmt]) -> bool { + !suite.iter().any(|statement| { + matches!(statement, Stmt::ImportFrom(import) + if import.module.as_ref().is_some_and(|module| module.as_str() == "__future__") + && import.names.iter().any(|alias| alias.name.as_str() == "annotations")) + }) +} + +/// whether this decorator was written with an `@` +/// +/// basedpython's class and function *modifiers* — `sealed`, `static`, `data class` and +/// the rest — reach the ast as decorators with no `@` in front of them, which is the +/// only thing that tells the two apart. the transpiler settles them by the same test. +/// +/// this matters more here than anywhere: a decorator becomes a **name looked up in the +/// module namespace at init**, and a modifier has no such name. `static def m` compiled +/// to `By_ApplyDecorator(dict, "m", "static")` and the whole extension then failed to +/// import with `NameError: name 'static' is not defined` +fn is_written_decorator( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> bool { + let source = ruff_db::source::source_text(db, model.file()); + source + .as_bytes() + .get(usize::from(decorator.range().start())) + .copied() + == Some(b'@') +} + +/// what a class decorator means to the native build +/// +/// a written `@` decorator is applied to the *namespace entry* after the type is +/// installed, exactly as the class statement would have; [`decorator_path`] says which +/// expressions still mean there what they meant where the `class` stood. a *modifier* is +/// a bare name and never anything else, which is why the two questions are asked in this +/// order +fn class_modifier( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> Lowered { + if is_written_decorator(db, model, decorator) { + return decorator_path(&decorator.expression).map(Modifier::Apply); + } + let Expr::Name(name) = &decorator.expression else { + return Err(Decline::new( + "only a plain-name class modifier is lowered yet", + )); + }; + match name.id.as_str() { + "data_class" | "frozen_data_class" => Ok(Modifier::DataClass), + // erased by the transpiler, so the interpreted twin carries nothing either. + // `sealed` grows a `__sealed_members__` tuple, but the module body the + // fallback runs is what writes it + "abstract" | "open" | "sealed" | "export" => Ok(Modifier::Erased), + // `final` becomes `@final` from `typing`, which returns its argument — the + // one class decorator whose result is provably the class it was handed + "final" => Ok(Modifier::Apply(Decorator::name("final"))), + // `private` renames the class and `protocol` rewrites its bases: neither is a + // decorator at all, and the emitted type would answer to the wrong name or + // stand outside the protocol it was declared to be + _ => Err(Decline::new( + "this class modifier changes what the class is, which an emitted type cannot follow", + )), + } +} + +/// the same for a function or a method +/// +/// the modifier mapping is the transpiler's: each of these becomes the named python +/// decorator in the interpreted twin, and the fallback preamble is what binds the name — +/// so looking it up in the module namespace at init finds exactly what the twin used +fn function_modifier( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> Lowered { + if is_written_decorator(db, model, decorator) { + return decorator_path(&decorator.expression).map(Modifier::Apply); + } + let Expr::Name(name) = &decorator.expression else { + return Err(Decline::new("only a plain-name modifier is lowered yet")); + }; + match name.id.as_str() { + "static" => Ok(Modifier::Apply(Decorator::name("staticmethod"))), + "classmethod" => Ok(Modifier::Apply(Decorator::name("classmethod"))), + "abstract" => Ok(Modifier::Apply(Decorator::name("abstractmethod"))), + "final" => Ok(Modifier::Apply(Decorator::name("final"))), + "override" => Ok(Modifier::Apply(Decorator::name("override"))), + // neither reaches the interpreted twin as a decorator + "open" | "export" => Ok(Modifier::Erased), + // `private` mangles the name the definition is bound under, which is a + // rename rather than a decorator + _ => Err(Decline::new( + "this modifier changes what the definition is bound as, not what it is", + )), + } +} + /// the class a class extends /// /// a name this module does not emit is still lowerable, however many of them: the type @@ -1741,6 +2493,7 @@ fn base_class( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, model: &SemanticModel<'_>, + suite: &[Stmt], class: &ast::StmtClassDef, layouts: &Layouts, ) -> Lowered> { @@ -1752,15 +2505,24 @@ fn base_class( // the bases external even when none were written: python supplies `(object,)` for an // empty one itself, the same as `type("C", (), ns)` does let keyed = !arguments.keywords.is_empty(); - match arguments.args.as_ref() { - [] if keyed => Ok(Some(ClassBase::External(Vec::new()))), - [] => Ok(None), + // what each base written as a plain name stands for — see `base_stands_for` + let named: Vec> = arguments + .args + .iter() + .map(|base| match base { + Expr::Name(name) => base_stands_for(suite, name.id.as_str()).map(Some), + _ => Ok(None), + }) + .collect::>>()?; + match (arguments.args.as_ref(), named.as_slice()) { + ([], _) if keyed => Ok(Some(ClassBase::External(Vec::new()))), + ([], _) => Ok(None), // `class C(object)` is what `class C:` already is — the base adds no storage // and no members, so there is nothing to lay out and nothing to inherit. // resolved rather than matched by name, because a module may bind `object` // to something else entirely - [base] if !keyed && is_builtin_object(db, env, model, base, layouts) => Ok(None), - [Expr::Name(name)] if layouts.contains_key(name.id.as_str()) => { + ([base], _) if !keyed && is_builtin_object(db, env, model, base, layouts) => Ok(None), + ([Expr::Name(_)], [Some(name)]) if layouts.contains_key(*name) => { if keyed { // the layout would have to be ours, which only the type spec lays out, // and a spec has nowhere to put a keyword @@ -1768,29 +2530,33 @@ fn base_class( "a class keyword on a base this module emits is not lowered yet", )); } - Ok(Some(ClassBase::InModule(name.id.to_string()))) + Ok(Some(ClassBase::InModule((*name).to_string()))) } // more than one base: python works out the mro and which of them owns the // layout, and this class declares none of its own. one of *ours* may stand // among them so long as it lays nothing out — it is in the module namespace by // the time this class is built, so it resolves like any other name, and having // no fields it asks for no room this class does not control - bases => { + (bases, named) => { let mut paths = Vec::with_capacity(bases.len()); - for base in bases { - if let Expr::Name(name) = base - && layouts - .get(name.id.as_str()) - .is_some_and(|fields| !fields.is_empty()) + for (base, name) in bases.iter().zip(named) { + if let Some(name) = name + && layouts.get(*name).is_some_and(|fields| !fields.is_empty()) { return Err(Decline::new( "a base this module lays out cannot stand beside one it does not", )); } - let Some(path) = dotted_path(base) else { - return Err(Decline::new( - "only a name or a dotted name is lowered as a base class yet", - )); + let path = match name { + Some(name) => (*name).to_string(), + None => match dotted_path(base) { + Some(path) => path, + None => { + return Err(Decline::new( + "only a name or a dotted name is lowered as a base class yet", + )); + } + }, }; if !external_base_resolves(model, base) { return Err(Decline::new( @@ -1804,6 +2570,109 @@ fn base_class( } } +/// the name a base written as a plain name stands for in this module's own namespace +/// +/// every question asked about a base is asked of the *name*: whether this module lays it +/// out, whether it stands beside one that does, and what the emitted module looks up at +/// import. a module-level alias answers all three about a name that is not the class. +/// +/// an emitted type is put in the namespace under the class's **own** name as it is +/// built, and an alias is carried over to it only once every class has been built — so +/// `Alias = Root` left `class C(Alias)` built on the interpreted definition while +/// `m.Root` was the emitted type, and `isinstance(C(), Root)` answered `False` where the +/// interpreter says `True`. the layout gates missed it for the same reason: +/// `class C(Alias, ABC)` compiled where `class C(Root, ABC)` is refused. +/// +/// only a name the module body binds exactly once, to another plain name, is followed, +/// and only as far as a class this module writes. a chain that leaves the module is left +/// where it was written: swapping one name the body may rebind for another buys nothing, +/// and both stand for the same object at import. +/// +/// a name bound twice stands for whichever binding ran last rather than the one the +/// class statement saw, which is a question about order that a name cannot answer — so +/// where any of those bindings is a class this module writes, the class declines +fn base_stands_for<'a>(suite: &'a [Stmt], written: &'a str) -> Lowered<&'a str> { + let mut current = written; + // bounded the way the base walks are: an alias chain cannot reach a name twice + // without being a cycle, and a cycle would otherwise spin here rather than settle + for _ in 0..=suite.len() { + if class_written(suite, current).is_some() { + return Ok(current); + } + match module_binding(suite, current) { + Bound::Loose => return Ok(written), + Bound::Alias(next) => current = next, + Bound::Contested => { + return Err(Decline::new( + "a base the module binds more than once stands for the class bound last, not the one it was built on", + )); + } + } + } + Ok(written) +} + +/// what this module's own body binds a base name to +enum Bound<'a> { + /// nothing here says, so the name stands for itself — an import, or a value no + /// class of this module's is behind + Loose, + /// the one module-level `name = other` + Alias(&'a str), + /// bound more than once, and a class this module writes is one of them + Contested, +} + +fn module_binding<'a>(suite: &'a [Stmt], name: &str) -> Bound<'a> { + let mut aliases = Vec::new(); + let mut bindings = 0usize; + for statement in suite { + let value = match statement { + Stmt::Assign(assign) => match assign.targets.as_slice() { + [Expr::Name(target)] if target.id.as_str() == name => Some(assign.value.as_ref()), + // a name among several targets is bound here too, so it counts as a + // binding even though this one does not say what to + targets if targets.iter().any(|target| binds_name(target, name)) => None, + _ => continue, + }, + // an annotation with no value binds nothing at all + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(target) if target.id.as_str() == name => match &assign.value { + Some(value) => Some(value.as_ref()), + None => continue, + }, + _ => continue, + }, + _ => continue, + }; + bindings += 1; + if let Some(Expr::Name(value)) = value { + aliases.push(value.id.as_str()); + } + } + match (bindings, aliases.as_slice()) { + (1, [alias]) => Bound::Alias(alias), + _ if aliases + .iter() + .any(|alias| class_written(suite, alias).is_some()) => + { + Bound::Contested + } + _ => Bound::Loose, + } +} + +/// whether an assignment target binds this name anywhere inside it +fn binds_name(target: &Expr, name: &str) -> bool { + match target { + Expr::Name(target) => target.id.as_str() == name, + Expr::Tuple(tuple) => tuple.iter().any(|element| binds_name(element, name)), + Expr::List(list) => list.iter().any(|element| binds_name(element, name)), + Expr::Starred(starred) => binds_name(&starred.value, name), + _ => false, + } +} + /// the keyword arguments a class header carries, as the emitter needs them /// /// the values are evaluated in the module scope at class-definition time, which at @@ -1989,25 +2858,8 @@ fn class_fields( "a `setattr` on the receiver names its attribute at runtime", )); } - let mut is_data = false; - // a real decorator is applied to the *namespace entry* after the type is - // installed, and every construction resolves the class through that namespace — - // so it gets whatever the decorator produced - let mut class_decorators = Vec::new(); - for decorator in &class.decorator_list { - match &decorator.expression { - Expr::Name(name) if matches!(name.id.as_str(), "data_class" | "frozen_data_class") => { - is_data = true; - } - Expr::Name(name) => class_decorators.push(name.id.to_string()), - _ => { - return Err(Decline::new( - "only a plain-name class decorator is lowered yet", - )); - } - } - } - let base = base_class(db, env, model, class, layouts)?; + let (is_data, _) = class_modifiers(db, model, class)?; + let base = base_class(db, env, model, suite, class, layouts)?; // a subclass's struct *begins* with its base's fields, in the same order and // unchanged, so a pointer to one is a valid pointer to the other — which is what @@ -2022,6 +2874,9 @@ fn class_fields( .flatten() .cloned() .collect(); + // the inherited ones come first and nothing after that removes or reorders one, so + // what this class adds of its own is whatever the field passes left past them + let taken = inherited.len(); let fields = if is_data { data_fields(db, env, model, class, layouts, inherited)? } else { @@ -2033,9 +2888,19 @@ fn class_fields( init_fields(db, env, model, class, layouts, inherited)?, )? }; - let fields = - spec_built_where_needed(db, env, model, suite, class, base.as_ref(), layouts, fields)?; - metaclass_carries_the_body(class, base.as_ref(), layouts, is_data)?; + // a class that adds no field of its own keeps what its base keeps, at the offsets + // the base laid them out, reached through the descriptors the base published — so + // there is no region past the base's instance for it to own, and none of the three + // slots that would reach one. it is built the way any other class with no storage of + // its own is, and what it declares here is the same nothing + let fields = if fields.len() == taken + && appends_past_a_base_of_ours(db, env, model, suite, base.as_ref(), layouts) + { + Vec::new() + } else { + spec_built_where_needed(db, env, model, suite, class, base.as_ref(), layouts, fields)? + }; + metaclass_carries_the_body(class, base.as_ref(), layouts)?; Ok(presence_where_a_finalizer_reads( db, env, model, suite, layouts, class, fields, )) @@ -2050,6 +2915,11 @@ fn class_fields( /// finished type, could have run. an `abstractmethod` there also raises, since a compiled /// method is a descriptor and takes no attributes. /// +/// a class-level constant is not in the same position, though it reads like it: it goes +/// into the namespace with the methods, and the class is asked afterwards whether it kept +/// the value — see `By_ConstantsHeldUp`. a decorator has no such answer, because what it +/// produces is only knowable by running it on a class that already exists. +/// /// this is asked while the layouts are still settling rather than while the body is /// lowered, and where it turns a class down that class leaves the layout set — so a /// subclass of one takes the same external base every other declining class's subclass @@ -2060,7 +2930,6 @@ fn metaclass_carries_the_body( class: &ast::StmtClassDef, base: Option<&ClassBase>, layouts: &Layouts, - is_data: bool, ) -> Lowered<()> { if class_keywords(class)?.is_empty() && !stands_on_an_emitted_base(base, layouts) { return Ok(()); @@ -2073,23 +2942,6 @@ fn metaclass_carries_the_body( "a decorated method on a class built through its metaclass is not lowered yet", )); } - // a class-level constant is settled after the metaclass has decided too, so a class - // with one is kept off that construction — and there is nowhere else for such a class - // to go. what would be left is the interpreted definition, which is only there while - // the module still holds the name: `ast` pops `Num` out of its own globals, so the - // class has to be declined here rather than left to fail at import. - // in a `data class` the annotations are the fields instead, and the layout has taken - // them - let constant = class.body.iter().any(|statement| match statement { - Stmt::AnnAssign(node) => !is_data && node.value.is_some(), - Stmt::Assign(_) => true, - _ => false, - }); - if constant { - return Err(Decline::new( - "a class-level constant on a class built through its metaclass is not lowered yet", - )); - } Ok(()) } @@ -2168,25 +3020,7 @@ fn spec_built_where_needed( "a class with fields of its own cannot have a base this module emits beside one it does not", )); } - // this class's fields sit past the base's instance, and reaching them takes three - // type slots of its own that call the base's. python's own three resolve which base - // to chain to from the instance's type rather than from the type that declared them, - // so they find this class's back and call it — a recursion that ends as a stack - // overflow. a class this module *writes* carries exactly those whichever way it is - // built: emitted from a spec, or left to the interpreted definition where it declined - let appended = match base { - None => false, - Some(ClassBase::External(_)) => true, - Some(ClassBase::InModule(name)) => { - laid_out_from_outside(db, env, model, suite, layouts, name) - } - }; - if appended - && base.is_some_and(|base| { - base.plain_names() - .any(|name| class_written(suite, name).is_some()) - }) - { + if appends_past_a_base_of_ours(db, env, model, suite, base, layouts) { return Err(Decline::new( "a class whose fields sit past a base's instance needs a base python frees itself, and one this module writes is not", )); @@ -2207,6 +3041,40 @@ fn spec_built_where_needed( Ok(fields) } +/// whether a class with storage of its own would keep it past an instance of a base this +/// module writes +/// +/// reaching such storage takes three type slots of this class's own that call the base's. +/// python's own three resolve which base to chain to from the instance's type rather than +/// from the type that declared them, so they find this class's back and call it — a +/// recursion that ends as a stack overflow. a class this module *writes* carries exactly +/// those whichever way it is built: emitted from a spec, or left to the interpreted +/// definition where it declined. +/// +/// a class that adds no field of its own asks nothing of this: there is no region past +/// the base's instance for it to own, so none of the three slots is its to supply +fn appends_past_a_base_of_ours( + db: &dyn ty_python_semantic::Db, + env: &ProgramEnvironment<'_>, + model: &SemanticModel<'_>, + suite: &[Stmt], + base: Option<&ClassBase>, + layouts: &Layouts, +) -> bool { + let appended = match base { + None => false, + Some(ClassBase::External(_)) => true, + Some(ClassBase::InModule(name)) => { + laid_out_from_outside(db, env, model, suite, layouts, name) + } + }; + appended + && base.is_some_and(|base| { + base.plain_names() + .any(|name| class_written(suite, name).is_some()) + }) +} + /// the class statement this module writes under a name, where it writes one /// /// asked of the *source* rather than of the emitted set, because what a base name stands @@ -2301,7 +3169,7 @@ fn layout_root( let mut current = class; // bounded the way [`laid_out_from_outside`] is for _ in 0..=suite.len() { - let next = base_class(db, env, model, current, layouts) + let next = base_class(db, env, model, suite, current, layouts) .ok() .flatten() .and_then(|base| base.in_module().map(str::to_owned)) @@ -2336,7 +3204,10 @@ fn laid_out_from_outside( // bounded by the class count: a base chain cannot visit one twice without being a // cycle, and a cycle would otherwise spin here rather than settle for _ in 0..=suite.len() { - match base_class(db, env, model, current, layouts).ok().flatten() { + match base_class(db, env, model, suite, current, layouts) + .ok() + .flatten() + { None => return false, Some(ClassBase::External(_)) => return true, Some(ClassBase::InModule(base)) => match class_written(suite, &base) { @@ -2631,9 +3502,11 @@ fn zero_argument_super( "python makes this method implicitly static or class, so slot zero holds the class rather than a receiver", ); } + // `static` and `classmethod` are how basedpython spells the first two, so the + // marker forms have to be read here as well as the `@` forms let rebinds_slot_zero = function.decorator_list.iter().any(|decorator| { matches!(&decorator.expression, Expr::Name(name) - if matches!(name.id.as_str(), "classmethod" | "staticmethod")) + if matches!(name.id.as_str(), "classmethod" | "staticmethod" | "static")) }); if rebinds_slot_zero { return Err( @@ -2649,6 +3522,64 @@ fn zero_argument_super( }) } +/// which of python's three method conventions this definition asks for, with the +/// decorator that asked for it taken off the list +/// +/// `staticmethod` and `classmethod` are not applied at module init like every other +/// decorator: the method table entry carries `METH_STATIC` or `METH_CLASS`, and the +/// type builds the descriptor python would have built. so honouring one means dropping +/// it, and the two must not both happen. +/// +/// that only holds where it is the **only** decorator. the runtime folds the rest onto +/// the attribute it reads back off the finished type — and reading a static method back +/// hands over the plain function, which would then be written back as an ordinary +/// method. a second decorator keeps the decline +fn method_binding( + function: &ast::StmtFunctionDef, + receiver: Option>, + decorators: &mut Vec, +) -> Lowered { + if !matches!(receiver, Some(Receiver::Explicit(_))) { + return Ok(Binding::Instance); + } + // only a bare name says which convention: `abc.abstractmethod` is not one however + // its last segment reads, and neither is any other attribute off something else + let convention = |decorator: &Decorator| match decorator.as_name()? { + "staticmethod" => Some(Binding::Static), + "classmethod" => Some(Binding::Class), + _ => None, + }; + let Some(binding) = decorators.iter().find_map(convention) else { + return Ok(Binding::Instance); + }; + if decorators.len() > 1 { + return Err(Decline::new( + "a second decorator over `classmethod` or `staticmethod` is folded onto the attribute read back off the type, which is no longer the one it was", + )); + } + // python already makes these implicitly static or class, and the emitted type + // publishes its own `__class_getitem__` for a generic class — so a table entry of + // our own would either double the convention or collide with that one + if matches!( + function.name.as_str(), + "__new__" | "__init_subclass__" | "__class_getitem__" + ) { + return Err(Decline::new( + "python gives this method a convention of its own, which a method table entry would duplicate", + )); + } + // a generator's state object is namespaced by the receiver's class, and neither of + // these has one — so two classes with a static `values` would want one state class + // between them + if generators::is_generator(&function.body) || function.is_async { + return Err(Decline::new( + "a `classmethod` or `staticmethod` that suspends is not lowered yet", + )); + } + decorators.clear(); + Ok(binding) +} + /// a method: an ordinary function whose exported name is namespaced by the class fn lower_method( unit: Unit<'_>, @@ -2724,16 +3655,50 @@ fn lower_function_with_receiver( signatures, .. } = unit; - // a decorator is applied at module init to the installed native function, so - // the body still compiles. only a plain name is supported: a call or an - // attribute would need arguments evaluated at init + // a name this frame declares `global` gets no register and no environment field: + // both halves of it are the module namespace, reached through `Place::Global`. + // keeping it out of the locals here is what makes that true — a register declared + // for it would be what a nested function captured, and the two would disagree + let declared_global = declared_globals(&function.body); + // a decorator is applied at module init to the installed native function, so the + // body still compiles. `decorator_path` says which expressions mean the same thing + // evaluated there as they did where the `def` stood. + // + // a *modifier* is not a name at all — it is spelled without an `@` and the + // transpiler rewrites it — so it is translated to whatever the interpreted twin + // ended up with, or dropped where the twin has nothing + // a *nested* function's decorators belong to the frame the `def` stands in, which + // applies them to the closure it just made — see `nested_def`. carrying them here + // as well would apply them a second time, to the environment class's method + let nested = matches!(receiver, Some(Receiver::Implicit(_))); let mut decorators = Vec::with_capacity(function.decorator_list.len()); - for decorator in &function.decorator_list { - let Expr::Name(name) = &decorator.expression else { - return Err(Decline::new("only a plain-name decorator is lowered yet")); - }; - decorators.push(name.id.to_string()); - } + for decorator in function.decorator_list.iter().filter(|_| !nested) { + match function_modifier(db, model, decorator)? { + Modifier::Apply(name) => decorators.push(name), + Modifier::Erased | Modifier::DataClass => {} + } + } + // a method's first parameter is forced to the receiver, because that is what python + // puts in slot zero — and `staticmethod` and `classmethod` are exactly the two that + // say it is not. the method table entry says which, so the decorator is honoured by + // the emitted type rather than applied to it, and comes off the list here + let binding = method_binding(function, receiver, &mut decorators)?; + if receiver.is_none() { + decorator_stays_unread(unit.read, function.name.as_str(), &decorators)?; + } + // a class method's slot zero holds the *class*: an ordinary object, and pointedly + // not an instance of the layout, so nothing reads a field off it. a static method + // has no slot zero at all, and its first written parameter keeps its own type + let class_object = RType::OBJECT; + // what the `def` was *written* as a method of, which a zero-argument `super()` is + // asked about — it has its own account of why neither of these fills slot zero with + // a receiver, and the effective one no longer says which class the method is on + let declared_receiver = receiver; + let receiver = match binding { + Binding::Instance => receiver, + Binding::Static => None, + Binding::Class => Some(Receiver::Explicit(&class_object)), + }; // a generator and a coroutine do not run their body when called: they allocate a // state object and hand it back. the body becomes a method of that object @@ -2764,10 +3729,21 @@ fn lower_function_with_receiver( computed_defaults, } = signature(db, env, model, function, layouts, receiver, arrays)?; + // a boundary that hands the call on takes the twin off the interpreted class, and + // for a class method that is already *bound* — to the interpreted class, not to the + // one in slot zero. handing it the class as well would give the body two of them + if binding == Binding::Class && !computed_defaults.is_empty() { + return Err(Decline::new( + "a `classmethod` whose default is not an immediate would reach a twin already bound to the interpreted class", + )); + } + // a nested function lives on a generated environment class, whose fields are // the captures. it has to exist before the body is lowered, because the `def` // statement allocates it - let locals_here = local_representations(db, env, model, &function.body, layouts, unit.arrays); + let mut locals_here = + local_representations(db, env, model, &function.body, layouts, unit.arrays); + locals_here.retain(|(name, _)| !declared_global.contains(name)); let bound: HashSet = params .iter() .map(|(name, _)| name.clone()) @@ -2840,7 +3816,15 @@ fn lower_function_with_receiver( .collect(); let split = !cells_here.is_empty() && !bindings_here.is_empty(); - let frame_name = closures::environment_name(enclosing, &function.name); + // the *name* is qualified by the class the `def` was written in even where the + // frame has no receiver of that class, or a static `parse` and a module-level + // `parse` would ask for one environment class between them. the chain is not: + // `enclosing` is what says whether there is an outer frame to reach through, and + // neither of these has one + let frame_name = closures::environment_name( + enclosing.or_else(|| unit.owner.filter(|_| binding != Binding::Instance)), + &function.name, + ); let frame_owned: HashSet = if split { owned.difference(&bindings_here).cloned().collect() } else { @@ -3088,7 +4072,7 @@ fn lower_function_with_receiver( (None, _) => None, }; - let zero_super = zero_argument_super(function, receiver); + let zero_super = zero_argument_super(function, declared_receiver); let mut lowering = Lowering { arrays: unit.arrays, @@ -3099,7 +4083,9 @@ fn lower_function_with_receiver( model, builder, locals, + globals: declared_global, native_callees, + decorated: unit.decorated, layouts, methods, signatures, @@ -3146,7 +4132,8 @@ fn lower_function_with_receiver( lowering.builder.terminate(Terminator::Return(value)); } - let lowered = lowering.builder.finish(); + let mut lowered = lowering.builder.finish(); + lowered.binding = binding; // the environment's methods are the nested bodies, lowered with the environment // as the receiver — so a captured read is a field read like any other let environments = match environment { @@ -3240,6 +4227,11 @@ struct Unit<'a> { db: &'a dyn ty_python_semantic::Db, model: &'a SemanticModel<'a>, native_callees: &'a HashSet, + /// the module-level functions whose name a decorator rebinds, so a call through + /// that name has to resolve it rather than reach the native entry + decorated: &'a HashSet, + /// every name the module reads anywhere — see [`names_read`] + read: &'a BTreeSet<&'a str>, /// the module body, so a class can be asked about a base's base — which is what /// says whether its own fields sit inside a base's instance or past one suite: &'a [Stmt], @@ -3283,6 +4275,16 @@ struct Captured { #[derive(Clone)] enum Place { Register(RegisterId), + /// a name this frame declares `global`: it lives in the module namespace, and + /// neither half of it is a register + /// + /// the declaration is not a hint the write can ignore. python's binding is the + /// module's, so a write is visible at once to every other reader — and *this* + /// frame's own later reads have to come back out of the namespace too, or the + /// two halves stop agreeing with each other rather than with the module + Global { + name: String, + }, /// a field of a receiver register: a capture neither frame writes, copied in /// where the `def` runs Field { @@ -4258,6 +5260,44 @@ fn buffer_safe(body: &[Stmt], name: &str, arrays: &ArrayEditions) -> bool { mentions.iter().all(|mention| safe.contains(mention)) } +/// every name any frame in this module declares `global`, at any depth +/// +/// a `global` declaration is written in order to rebind: the name stops holding what +/// the module bound at import, and a call or a construction through it has to find +/// what is really there. that is exactly what a decorator does to a name, so the two +/// share a set — see `decorated` in [`build_module`]. +/// +/// a declaration with nothing assigned under it would be harmless, and is also +/// pointless, so this does not try to tell the two apart: naming one name too many +/// costs the direct call and nothing else, where naming one too few is a call that +/// reaches a definition the namespace no longer holds +fn declared_global_anywhere(body: &[Stmt]) -> HashSet { + let mut out = declared_globals(body); + for stmt in walk(body) { + match stmt { + Stmt::FunctionDef(node) => out.extend(declared_global_anywhere(&node.body)), + Stmt::ClassDef(node) => out.extend(declared_global_anywhere(&node.body)), + _ => {} + } + } + out +} + +/// the names a frame declares `global` +/// +/// [`walk`] stops at a nested `def` or `class`, which is what makes this per-scope: +/// a declaration inside one is that scope's, and python does not pass it outwards +fn declared_globals(body: &[Stmt]) -> HashSet { + walk(body) + .into_iter() + .filter_map(|stmt| match stmt { + Stmt::Global(node) => Some(node.names.iter().map(ast::Identifier::to_string)), + _ => None, + }) + .flatten() + .collect() +} + /// every statement in `body`, including nested ones fn walk(body: &[Stmt]) -> Vec<&Stmt> { let mut out = Vec::new(); @@ -4299,7 +5339,12 @@ struct Lowering<'a, 'db> { model: &'a SemanticModel<'db>, builder: FunctionBuilder, locals: HashMap, + /// the names this frame declares `global`, which live in the module namespace + /// rather than in any register of this frame + globals: HashSet, native_callees: &'a HashSet, + /// the module-level functions whose name a decorator rebinds + decorated: &'a HashSet, layouts: &'a Layouts, methods: &'a Methods, /// the signature of each module-level function, so a call coerces its arguments @@ -4490,6 +5535,21 @@ impl Lowering<'_, '_> { name: self.attribute_name(&attribute.attr), }); } + // a name in the module namespace *does* have an unbound state: + // it is not in the dict. a register does not, which is why the + // rest of this arm still declines + Expr::Name(name) + if matches!( + self.place(name.id.as_str()), + Some(Place::Global { .. }) + ) => + { + let status = self.builder.temp(RType::BIT); + self.builder.push(Op::DeleteGlobal { + dest: status, + name: name.id.to_string(), + }); + } _ => { return Err(Decline::new( "`del` on a plain name is not lowered yet — a register \ @@ -6304,6 +7364,13 @@ impl Lowering<'_, '_> { /// a register wins over a field: a generator's *parameters* are registers even /// where its locals are fields, and a closure's own parameters shadow a capture fn place(&self, name: &str) -> Option { + // asked first, and it has to be: a `global` declaration says this name is not + // this frame's to bind, so nothing else may answer for it + if self.globals.contains(name) { + return Some(Place::Global { + name: name.to_string(), + }); + } if let Some(&id) = self.locals.get(name) { return Some(Place::Register(id)); } @@ -6379,6 +7446,16 @@ impl Lowering<'_, '_> { let ty = self.register_type(*id)?; Ok((Value::Register(*id), ty)) } + // resolved out of the namespace exactly as an undeclared name is, which is + // what keeps a read after a write in the same frame seeing the write + Place::Global { name } => { + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::LoadGlobal { + dest, + name: name.clone(), + }); + Ok((Value::Register(dest), RType::OBJECT)) + } Place::Field { receiver, class, @@ -6443,6 +7520,17 @@ impl Lowering<'_, '_> { fn write_place(&mut self, place: &Place, value: Value, ty: &RType) -> Lowered<()> { match place { Place::Register(id) => self.store(*id, value, ty), + // the namespace holds objects, so an unboxed value is boxed on the way in + Place::Global { name } => { + let value = self.widen_to_object(value, ty); + let status = self.builder.temp(RType::BIT); + self.builder.push(Op::StoreGlobal { + dest: status, + name: name.clone(), + value, + }); + Ok(()) + } Place::Field { receiver, class, @@ -6783,11 +7871,21 @@ impl Lowering<'_, '_> { /// binding and unboxes the result back to an instance pointer. none of that /// says anything a direct allocation and a native `__init__` does not. /// + /// a **decorated** class is the one this does not answer for. the decorator is + /// applied to the namespace entry, and a construction is written against that name + /// — so allocating the emitted layout skips the decorator entirely, and a decorator + /// that returns another class had every construction in the module building the + /// wrong object. that one has to go out through the namespace and find what is + /// really there. + /// /// only a plain positional call: a default or a keyword needs the binding the /// signature describes, and falling back to the interpreted path for those is /// correct — just slower fn construct(&mut self, name: &str, node: &ast::ExprCall) -> Lowered> { - if !self.layouts.contains_key(name) || !node.arguments.keywords.is_empty() { + if !self.layouts.contains_key(name) + || self.decorated.contains(name) + || !node.arguments.keywords.is_empty() + { return Ok(None); } let Some(signature) = self.signatures.get(&qualify(Some(name), "__init__")) else { @@ -7277,17 +8375,27 @@ impl Lowering<'_, '_> { // is applied last — the same order the `def` statement itself applies them let mut made = Value::Register(closure); for decorator in node.decorator_list.iter().rev() { - let Expr::Name(name) = &decorator.expression else { - return Err(Decline::new( - "only a plain-name decorator on a nested function is lowered yet", - )); - }; let wrapped = self.builder.temp(RType::OBJECT); - self.builder.push(Op::CallPython { - dest: wrapped, - callee: name.id.to_string(), - args: vec![made], - }); + // the decorator expression is evaluated *here*, where the `def` stands, in + // this frame — which is what python does and what makes an arbitrary + // expression safe to take: `@functools.wraps(func)` reads `func` out of + // this frame's own registers, at the moment the closure is made + if let Expr::Name(name) = &decorator.expression + && !self.binds(name.id.as_str()) + { + self.builder.push(Op::CallPython { + dest: wrapped, + callee: name.id.to_string(), + args: vec![made], + }); + } else { + let callee = self.callable(&decorator.expression)?; + self.builder.push(Op::CallValue { + dest: wrapped, + callee, + args: vec![made], + }); + } made = Value::Register(wrapped); } let dest_ty = self.register_type(dest)?; @@ -7609,11 +8717,12 @@ impl Lowering<'_, '_> { Expr::Name(node) => { let name = node.id.as_str(); match self.place(name) { - Some(place) => self.read_place(&place), - // a name this frame does not have is a global, resolved the way - // `LOAD_GLOBAL` resolves it. the result is an `object`, so the - // checker's type for the expression decides any narrowing - None => { + Some(Place::Global { .. }) | None => { + // a name this frame does not have is a global, resolved the way + // `LOAD_GLOBAL` resolves it — and a name it *declares* `global` + // is the same read, which is what makes the declaration mean + // anything. the result is an `object`, so the checker's type for + // the expression decides any narrowing let dest = self.builder.temp(RType::OBJECT); self.builder.push(Op::LoadGlobal { dest, @@ -7621,6 +8730,7 @@ impl Lowering<'_, '_> { }); self.narrow_call_result(dest, expr) } + Some(place) => self.read_place(&place), } } // a `yield` is a field write and a return. the code after it becomes a new @@ -9112,11 +10222,13 @@ impl Lowering<'_, '_> { { return self.call_unpacked(node); } - // keywords the compiler cannot bind here — a method, or a name the unit does - // not own — are bound by python, from a tuple and a dict + // keywords the compiler cannot bind here — a method, a name the unit does not + // own, or one a decorator rebound to a signature this unit never saw — are + // bound by python, from a tuple and a dict if !node.arguments.keywords.is_empty() && !matches!(node.func.as_ref(), Expr::Name(name) - if self.native_callees.contains(name.id.as_str())) + if self.native_callees.contains(name.id.as_str()) + && !self.decorated.contains(name.id.as_str())) { return self.call_unpacked(node); } @@ -9223,8 +10335,16 @@ impl Lowering<'_, '_> { // a name the unit does not own is resolved and called the way the // interpreter would, with everything boxed on both sides. a call the native - // entry cannot take goes the same way, and reaches the deferring boundary - if !self.native_callees.contains(name) || self.defers_call(name, node) { + // entry cannot take goes the same way, and reaches the deferring boundary. + // + // a *decorated* one goes the same way for a different reason: the name holds + // what the decorator returned, and the native entry is what it was handed. + // reaching it directly would skip the decorator entirely — which is a wrong + // answer rather than a missed optimization + if !self.native_callees.contains(name) + || self.decorated.contains(name) + || self.defers_call(name, node) + { let mut args = Vec::with_capacity(node.arguments.args.len()); for argument in &node.arguments.args { let (value, ty) = self.expression(argument)?; diff --git a/crates/by_irbuild/src/single_file.rs b/crates/by_irbuild/src/single_file.rs index 3a9f705e50..9e1042e5dc 100644 --- a/crates/by_irbuild/src/single_file.rs +++ b/crates/by_irbuild/src/single_file.rs @@ -64,7 +64,7 @@ fn make_db(source: &str, language: Language) -> (TestDb, File) { /// lower source into a module named `module_name` pub fn module_from_source( source: &str, - module_name: &str, + module_name: impl Into, language: Language, ) -> by_ir::function::ModuleIr { with_source_in(source, language, |db, env, model, suite| { diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index a14b7fee5c..b74ae14c0b 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -22,6 +22,15 @@ fn has_op(function: &by_ir::function::Function, predicate: impl Fn(&Op) -> bool) .any(predicate) } +/// each decorator as it was written, which is what a test about *which* decorators +/// travel with a definition wants to read +fn dotted(decorators: &[by_ir::function::Decorator]) -> Vec { + decorators + .iter() + .map(by_ir::function::Decorator::dotted) + .collect() +} + /// lower `source` and render the module's IR, failing if it does not verify fn ir(source: &str) -> String { with_source(source, |db, env, model, suite| { @@ -1038,6 +1047,34 @@ class Tagged: ); } +#[test] +fn a_decorated_class_with_a_class_level_constant_is_lowered() { + // the constant is copied off the body the interpreted `class` statement wrote, which + // is captured while that statement runs and so predates every decorator. it used to + // be read back off the finished definition, and a decorator that makes something of + // what the body wrote leaves that definition saying something else — `@dataclass` + // deletes the `field(init=False)` a body wrote. so this whole shape declined, which + // over the corpus was almost every decorated class there was + let reasons = declines( + "\ +def tagger(cls): + return cls + + +@tagger +class Tagged: + KIND: str = \"class-level\" + + def read(self) -> str: + return \"read\" +", + ); + assert!( + !reasons.iter().any(|(name, _)| name == "Tagged"), + "{reasons:?}" + ); +} + #[test] fn a_private_class_constant_takes_its_mangled_name() { // python binds `__params` written in `class Function` as `_Function__params`, so @@ -1256,17 +1293,17 @@ class Held: } #[test] -fn a_class_level_constant_under_a_class_keyword_declines() { - // a constant keeps its class off the metaclass construction, because it is settled - // after the metaclass has already decided what the class defines. a class keyword - // leaves nowhere else to go — a type spec has nowhere to put the keyword — so what - // would answer is the interpreted definition, and that is only there while the - // module still holds the name. `ast` pops `Num` straight out of its own globals, so - // leaving this to the runtime turns the import into a `NameError`. +fn a_class_level_constant_under_a_class_keyword_is_carried_not_declined() { + // a constant used to keep its class off the metaclass construction, on the reasoning + // that it is settled after the metaclass has decided what the class defines. it is + // not: it goes into the namespace with the methods, and the class is asked afterwards + // whether it kept the value. so the class lowers, carrying the constant. // - // `Plain` is the boundary: the same keyword with no constant still lowers - let reasons = declines( - "\ + // a decorated method is the boundary, and the one thing this gate still turns down — + // what a decorator produces is only knowable by running it on a finished class + assert_eq!( + class_constants( + "\ from abc import ABCMeta @@ -1275,23 +1312,61 @@ class Tagged(metaclass=ABCMeta): def label(self) -> str: return \"tagged\" - - -class Plain(metaclass=ABCMeta): - def label(self) -> str: - return \"plain\" ", + "Tagged" + ), + vec!["TAG".to_string()] ); assert_eq!( - reasons, + declines( + "\ +from abc import ABCMeta + + +class Decorated(metaclass=ABCMeta): + @staticmethod + def label() -> str: + return \"decorated\" +" + ), vec![( - "Tagged".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" + "Decorated".to_string(), + "a decorated method on a class built through its metaclass is not lowered yet" .to_string() )] ); } +#[test] +fn a_class_level_constant_beside_a_base_of_ours_keeps_that_base_emitted() { + // the shape the stdlib is made of: no keyword at all, a base this module emits + // standing beside one from outside, and a constant. a spec cannot work that base list + // out, so the metaclass is what builds it — and the constant rides into the namespace + // it is handed rather than closing it. + // + // the base staying emitted is the point, and it is what the decline used to cost: + // `Reader` declining took `Codec` with it, because the interpreted `Reader` extends + // the *twin's* `Codec` and `issubclass(m.Reader, m.Codec)` would answer False against + // a twin that says True. neither declines now + const SOURCE: &str = "\ +import codecs + + +class Codec(codecs.Codec): + def label(self) -> str: + return \"codec\" + + +class Reader(Codec, codecs.StreamReader): + tag = 1 + + def kind(self) -> str: + return \"reader\" +"; + assert_eq!(declines(SOURCE), vec![]); + assert_eq!(class_constants(SOURCE, "Reader"), vec!["tag".to_string()]); +} + #[test] fn a_base_this_module_lays_out_declines_beside_one_it_does_not() { // a class holding both kinds of base takes its whole layout from outside, so a base @@ -1339,6 +1414,187 @@ class OnFieldless(Fieldless, codecs.Codec): ); } +#[test] +fn a_base_written_as_an_alias_is_the_class_it_was_bound_to() { + // the alias is not a base out of this module: it stands for a class this module + // writes, and the emitted type is what the name will hold. taking it as external + // built the class on the interpreted definition instead — the alias is carried over + // to the emitted type only once every class has been built — so `isinstance` said + // `False` where the interpreter says `True` + const SOURCE: &str = "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![ + ("Root".to_string(), None), + ( + "Over".to_string(), + Some(ClassBase::InModule("Root".to_string())) + ) + ] + ); +} + +#[test] +fn an_alias_does_not_hide_a_base_this_module_lays_out() { + // the same refusal as the direct spelling, which is the point: every question the + // base list is asked is asked of the *name*, so an alias that stood for itself + // walked straight past this gate and compiled the one shape it exists to refuse + let reasons = declines( + "\ +import codecs + + +class Laid: + def __init__(self, n: int) -> None: + self.n = n + + +Alias = Laid + + +class OnLaid(Alias, codecs.Codec): + pass +", + ); + assert_eq!( + reasons, + vec![ + ( + "OnLaid".to_string(), + "a base this module lays out cannot stand beside one it does not".to_string() + ), + ( + "Laid".to_string(), + "`OnLaid` declined, so it extends the interpreted definition rather than this type" + .to_string() + ) + ] + ); +} + +#[test] +fn a_name_the_module_binds_twice_declines_rather_than_pick_a_class() { + // `Over` was built on `Root` and the name holds `Other` by the time the module body + // ends, so neither class is the answer: the one the class statement saw is gone, and + // the one the emitted module would look up is not what it extends + let reasons = declines( + "\ +class Root: + def root(self) -> str: + return \"root\" + + +class Other: + def other(self) -> str: + return \"other\" + + +Alias = Root +Alias = Other + + +class Over(Alias): + def side(self) -> str: + return \"over\" +", + ); + assert_eq!( + reasons, + vec![( + "Over".to_string(), + "a base the module binds more than once stands for the class bound last, not the one it was built on" + .to_string() + )] + ); +} + +#[test] +fn an_alias_chain_that_leaves_the_module_is_left_where_it_was_written() { + // the emitted module looks the base up by name, and both names hold the same object + // at import — so following one is no gain, and it would trade a name this body binds + // once for one it may bind again. only a chain that ends at a class of *ours* moves + const SOURCE: &str = "\ +from codecs import Codec + +Alias = Codec + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Over".to_string(), + Some(ClassBase::External(vec!["Alias".to_string()])) + )] + ); +} + +#[test] +fn a_name_bound_twice_to_nothing_of_ours_still_stands_for_itself() { + // the boundary: a name the module rebinds is a hazard only where a class of this + // module's is behind it. two imported names leave the base exactly what it was + const SOURCE: &str = "\ +import codecs + +Alias = codecs.Codec +Alias = codecs.StreamWriter + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Over".to_string(), + Some(ClassBase::External(vec!["Alias".to_string()])) + )] + ); +} + #[test] fn a_class_with_fields_of_its_own_declines_beside_a_base_this_module_emits() { // the other half: the base of ours lays nothing out, but *this* class does. only a @@ -1639,14 +1895,205 @@ class Beside(Rooted): } #[test] -fn an_annotated_class_attribute_under_a_class_keyword_declines_with_the_rest() { - // an annotated assignment is a class-level constant, so it reaches the same gate a - // plain one does. this is what making the annotation a binding costs: a class the - // compiler used to build silently without the attribute now refuses to build at all. - // over the stdlib that cost was nothing — every class this reason reaches was - // already declining for another - let reasons = declines( +fn a_subclass_that_appends_nothing_past_a_base_declares_nothing() { + // the fields are what makes the difference: `Held` above appends storage past a + // `Wrapper` instance and has no construction, while a class that adds *no* field of + // its own appends nothing at all. what such a class keeps is what `Wrapper` already + // keeps, at the offsets `Wrapper` laid them out and through the descriptors `Wrapper` + // published — so it is built the way any other class with no storage of its own is + // + // `Restating` is the same class written the other way round: assigning an attribute + // the base already stores adds nothing either, and the write lands on the base's + // field through the base's own setter + let (declined, layouts) = with_source( "\ +class Wrapper(OSError): + def __init__(self, code: int) -> None: + self.code = code + + +class Plain(Wrapper): + pass + + +class Tagged(Wrapper): + TAG = 1 + + +class Restating(Wrapper): + def __init__(self, code: int) -> None: + self.code = code + 1 +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + let declined: Vec<(String, String)> = module + .declined + .iter() + .map(|declined| (declined.name.clone(), declined.reason.clone())) + .collect(); + let layouts: Vec<(String, Vec)> = module + .classes + .iter() + .map(|class| { + ( + class.name.clone(), + class + .fields + .iter() + .map(|field| field.name.clone()) + .collect(), + ) + }) + .collect(); + (declined, layouts) + }, + ); + assert_eq!(declined, Vec::new()); + assert_eq!( + layouts, + vec![ + ("Wrapper".to_string(), vec!["code".to_string()]), + ("Plain".to_string(), Vec::new()), + ("Tagged".to_string(), Vec::new()), + ("Restating".to_string(), Vec::new()), + ] + ); +} + +#[test] +fn a_subclass_with_no_storage_is_rebuilt_on_a_base_that_declines_later() { + // a base is settled as one of ours while the layouts settle, and only the body being + // lowered can turn it down after that. a class with no storage of its own does not + // need it to have stayed one: what stands under the name at import is a class either + // way, and building on the *name* is what every class over a base out of this module + // already does — so it takes that construction rather than cascading behind the base. + // + // `Storing` is the boundary: a class with a field declares a size of its own, and a + // class over a base out of this module declares none — the base allocates. so the + // storage would have nowhere to go, and it cascades behind the base instead + const SOURCE: &str = "\ +class Base: + def __new__(cls) -> \"Base\": + return object.__new__(cls) + + def label(self) -> str: + return \"base\" + + +class Below(Base): + def side(self) -> str: + return \"below\" + + +class Storing(Base): + def __init__(self, extra: int) -> None: + self.extra = extra +"; + assert_eq!( + declines(SOURCE), + vec![ + ( + "Base".to_string(), + "`__new__` fills a type slot with no adapter yet".to_string() + ), + ( + "Storing".to_string(), + "`Base` declined, so it is not a base to build on".to_string() + ) + ] + ); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Below".to_string(), + Some(ClassBase::External(vec!["Base".to_string()])) + )] + ); +} + +#[test] +fn a_class_a_subclass_stores_inside_is_not_rebuilt_on_its_declining_base() { + // rebuilding `Middle` on a name would move the layout of everything under it outside + // the module, and `Storing`'s field would go from sitting inside a `Middle` instance + // to sitting past one — a construction that has no answer, and one that refuses the + // *whole module* at import rather than the class. `urllib.request` lost all nineteen + // of its compiled functions that way. + // + // `Aside` is the boundary: nothing stores anything under it, so it is rebuilt + const SOURCE: &str = "\ +class Base: + def __new__(cls) -> \"Base\": + return object.__new__(cls) + + def label(self) -> str: + return \"base\" + + +class Middle(Base): + def side(self) -> str: + return \"middle\" + + +class Storing(Middle): + def __init__(self) -> None: + self.extra = 1 + + +class Aside(Base): + def side(self) -> str: + return \"aside\" +"; + assert_eq!( + declines(SOURCE), + vec![ + ( + "Base".to_string(), + "`__new__` fills a type slot with no adapter yet".to_string() + ), + ( + "Middle".to_string(), + "`Base` declined, so it is not a base to build on".to_string() + ), + ( + "Storing".to_string(), + "`Middle` declined, so it is not a base to build on".to_string() + ) + ] + ); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Aside".to_string(), + Some(ClassBase::External(vec!["Base".to_string()])) + )] + ); +} + +#[test] +fn an_annotated_class_attribute_under_a_class_keyword_is_carried_with_the_rest() { + // an annotated assignment is a class-level constant, so it takes the same route a + // plain one does — into the namespace the metaclass is handed. it used to reach the + // same gate instead, and a class the compiler had been building silently without the + // attribute went from missing the attribute to refusing to build at all + assert_eq!( + class_constants( + "\ from abc import ABCMeta @@ -1656,24 +2103,23 @@ class Tagged(metaclass=ABCMeta): def label(self) -> str: return \"tagged\" ", - ); - assert_eq!( - reasons, - vec![( - "Tagged".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" - .to_string() - )] + "Tagged" + ), + vec!["TAG".to_string()] ); } #[test] -fn a_subclass_of_a_class_the_metaclass_gates_turn_down_builds_on_the_interpreted_base() { - // both gates are asked while the layouts settle, so a class either of them turns - // down leaves the layout set — and its subclass then takes the external base every - // other declining class's subclass takes rather than being laid out on a base - // nothing emits. asked while the *body* was lowered instead, the base stayed in the - // set and both subclasses cascaded behind it +fn a_subclass_of_a_class_the_metaclass_gate_turns_down_builds_on_the_interpreted_base() { + // the gate is asked while the layouts settle, so a class it turns down leaves the + // layout set — and its subclass then takes the external base every other declining + // class's subclass takes rather than being laid out on a base nothing emits. asked + // while the *body* was lowered instead, the base stayed in the set and the subclass + // cascaded behind it. + // + // `Constant` is the boundary in the other direction: a class-level constant no longer + // turns a class down, so that half stays in the layout set and its subclass is laid + // out on it — an `InModule` base against an `External` one const SOURCE: &str = "\ from abc import ABCMeta @@ -1702,21 +2148,15 @@ class BelowConstant(Constant): "; assert_eq!( declines(SOURCE), - vec![ - ( - "Decorated".to_string(), - "a decorated method on a class built through its metaclass is not lowered yet" - .to_string() - ), - ( - "Constant".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" - .to_string() - ) - ] + vec![( + "Decorated".to_string(), + "a decorated method on a class built through its metaclass is not lowered yet" + .to_string() + )] ); - // the base each subclass gets is the point: an `InModule` one would name a type - // this module never emits + // the base each subclass gets is the point: an `InModule` one below `Decorated` would + // name a type this module never emits, and an `External` one below `Constant` would + // give up a layout the module does have let bases = with_source(SOURCE, |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); module @@ -1732,9 +2172,14 @@ class BelowConstant(Constant): "BelowDecorated".to_string(), Some(ClassBase::External(vec!["Decorated".to_string()])) ), + // a keyword-only class header has no bases at all + ( + "Constant".to_string(), + Some(ClassBase::External(Vec::new())) + ), ( "BelowConstant".to_string(), - Some(ClassBase::External(vec!["Constant".to_string()])) + Some(ClassBase::InModule("Constant".to_string())) ) ] ); @@ -2060,13 +2505,11 @@ def plain(p: Plain) -> int: #[test] fn a_final_receiver_keeps_its_direct_call() { - // AB3 and AB4 traded the direct call away for any class that is decorated or in - // an inheritance chain. `@final` is about the *place*: nothing can subclass it, - // so there is no override for the protocol to find + // AB3 and AB4 traded the direct call away for any class in an inheritance chain. + // `final` is about the *place*: nothing can subclass it, so there is no override + // for the protocol to find with_source( "\ -from typing import final - data class Open: n: int @@ -2076,8 +2519,7 @@ data class Open: data class Derived(Open): extra: int -@final -data class Fixed(Open): +final data class Fixed(Open): label: str def tripled(self) -> int: @@ -2162,23 +2604,31 @@ def through(s: Shape) -> str: } #[test] -fn a_decorated_class_gives_up_its_direct_call() { +fn a_decorated_class_gives_up_its_direct_call_and_its_direct_construction() { // a decorated class is a mutable heap type, and python can rebind a method on - // one — a direct call would not see the rebinding + // one — a direct call would not see the rebinding. + // + // the *construction* goes the same way, and for a sharper reason: the decorator + // replaces what the module namespace binds, so `Loud(...)` names whatever it + // returned. allocating the emitted layout instead skipped the decorator outright, + // and a decorator returning another class had every construction in the module + // building the wrong object with no diagnostic at all with_source( "\ def tagged(cls: type) -> type: return cls @tagged -data class Loud: - n: int +class Loud: + def __init__(self, n: int) -> None: + self.n = n def doubled(self) -> int: return self.n * 2 -data class Quiet: - n: int +class Quiet: + def __init__(self, n: int) -> None: + self.n = n def doubled(self) -> int: return self.n * 2 @@ -2188,6 +2638,12 @@ def loud(x: Loud) -> int: def quiet(x: Quiet) -> int: return x.doubled() + +def build_loud(n: int) -> int: + return Loud(n).doubled() + +def build_quiet(n: int) -> int: + return Quiet(n).doubled() ", |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); @@ -2211,17 +2667,65 @@ def quiet(x: Quiet) -> int: "{}", ops("loud") ); + // and the undecorated one is allocated where it stands, while the + // decorated one is resolved through the namespace the decorator wrote to + assert!( + ops("build_quiet").contains("new Quiet"), + "{}", + ops("build_quiet") + ); + assert!( + !ops("build_loud").contains("new Loud"), + "{}", + ops("build_loud") + ); + assert!( + ops("build_loud").contains("pycall Loud"), + "{}", + ops("build_loud") + ); // and its decorators travel with the class let loud = module .classes .iter() .find(|class| class.name == "Loud") .expect("Loud is emitted"); - assert_eq!(loud.decorators, ["tagged"]); + assert_eq!(dotted(&loud.decorators), ["tagged"]); }, ); } +#[test] +fn a_class_modifier_is_not_a_decorator() { + // `sealed`, `abstract`, `open` and `export` reach the ast as decorators with no + // `@`, and the transpiler erases them — so the interpreted twin has no such name + // and looking one up at module init raised `NameError` and took the whole + // extension down with it + for modifier in ["sealed", "abstract", "open", "export"] { + let source = format!( + "\ +{modifier} data class Shape: + n: int +" + ); + let decorators = with_source(&source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{modifier}: {:?}", + module.declined + ); + module + .classes + .iter() + .find(|class| class.name == "Shape") + .map(|class| dotted(&class.decorators)) + .unwrap_or_else(|| panic!("{modifier}: Shape is emitted")) + }); + assert!(decorators.is_empty(), "{modifier}: {decorators:?}"); + } +} + #[test] fn a_class_with_a_hand_written_dunder_is_declined() { // `__init__` is generated from the fields, so a hand-written one would @@ -2246,82 +2750,518 @@ data class Point: } #[test] -fn len_is_an_intrinsic_not_a_call() { - let ir = ir("def f(s: str) -> int:\n return len(s)\n"); - assert!(ir.contains("= len "), "{ir}"); - assert!(!ir.contains("call len"), "{ir}"); +fn len_is_an_intrinsic_not_a_call() { + let ir = ir("def f(s: str) -> int:\n return len(s)\n"); + assert!(ir.contains("= len "), "{ir}"); + assert!(!ir.contains("call len"), "{ir}"); +} + +#[test] +fn a_module_defining_len_shadows_the_intrinsic() { + let ir = ir("\ +def len(x: int) -> int: + return x + +def f(a: int) -> int: + return len(a) +"); + assert!(ir.contains("call len(a)"), "{ir}"); +} + +#[test] +fn concatenating_two_strings_stays_a_string() { + let ir = ir("def f(a: str, b: str) -> str:\n return a + b\n"); + assert!(ir.contains("-> str"), "{ir}"); + assert!(ir.contains(" ++ "), "{ir}"); +} + +#[test] +fn an_async_function_is_declined() { + assert!(decline("def f(a: int) -> None:\n try:\n pass\n except* ValueError:\n pass\n") + .contains("`except*`")); +} + +#[test] +fn a_decorated_function_still_compiles() { + // the decorator is applied at module init to the installed native function, + // so the body is compiled either way + let source = "\ +def deco(f: object) -> object: + return f + +@deco +def f() -> None: + pass +"; + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let decorated = module + .functions + .iter() + .find(|function| function.name == "f") + .expect("f is compiled"); + assert_eq!(dotted(&decorated.decorators), ["deco"]); + }); +} + +#[test] +fn a_decorator_that_is_a_call_is_declined() { + // python calls `make(1)` where the `def` stands. module-level code is not compiled, + // so the only moment init has is the end of the module — by which time the + // interpreted twin has already made that call, and making it again would be a + // second one, in the wrong place + let source = "\ +def make(n: int) -> object: + return n + +@make(1) +def f() -> None: + pass +"; + let reason = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .declined + .iter() + .find(|declined| declined.name == "f") + .map(|declined| declined.reason.clone()) + .unwrap_or_default() + }); + assert!(reason.contains("run it a second time"), "{reason}"); +} + +#[test] +fn a_decorator_written_as_a_path_keeps_its_segments() { + // every step of `functools.cache` is a read, so evaluating it at init means what it + // meant where the `def` stood — and the ir carries the chain rather than one name + let source = "\ +import functools + +@functools.cache +def f(n: int) -> int: + return n +"; + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let decorated = module + .functions + .iter() + .find(|function| function.name == "f") + .expect("f is compiled"); + assert_eq!( + decorated.decorators, + [by_ir::function::Decorator::Path { + root: "functools".to_string(), + attributes: vec!["cache".to_string()], + }] + ); + }); +} + +#[test] +fn a_decorator_rooted_in_the_class_body_is_declined() { + // a decorator is resolved out of the *module* namespace at init, and a class body is + // not that namespace. `@x.setter` is the shape this exists for, but that one also + // writes two `def`s of one name — so the names here are distinct, or the duplicate + // would answer first and this guard would never be reached + let source = "\ +class Box: + def wrap(fn: object) -> object: + return fn + + @wrap + def value(self) -> int: + return 1 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.classes.iter().all(|class| class.name != "Box"), + "Box must not be emitted" + ); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert!( + reasons + .iter() + .any(|reason| reason.contains("`wrap` is bound by the class body")), + "{reasons:?}" + ); +} + +#[test] +fn a_decorated_function_is_not_called_at_its_native_entry() { + // the module namespace holds what the decorator returned, and the native entry is + // what it was handed — so reaching the entry directly runs the *undecorated* + // function. `caller` answered 2 where the interpreted module answered 4 + let ir = ir("\ +def double(fn: object) -> object: + def inner(x: int) -> int: + return fn(x) * 2 + return inner + +@double +def f(x: int) -> int: + return x + 1 + +def caller(x: int) -> int: + return f(x) + +def plain(x: int) -> int: + return x + 1 + +def other(x: int) -> int: + return plain(x) +"); + assert!(ir.contains("pycall f("), "{ir}"); + assert!(!ir.contains("= call f("), "{ir}"); + // an undecorated sibling still reaches its native entry + assert!(ir.contains("= call plain("), "{ir}"); +} + +#[test] +fn a_modifier_is_translated_rather_than_looked_up() { + // a modifier reaches the ast as a decorator with no `@`, and there is no such name + // in the module namespace to look up: `static` compiled to + // `By_ApplyDecorator(dict, "make", "static")` and the extension then failed to + // import with `NameError: name 'static' is not defined`. + // + // the transpiler rewrites each of these to a python decorator, and this is the + // same mapping — so the compiled definition ends up wearing what the interpreted + // twin wears + for (modifier, expected) in [ + ("abstract", vec!["abstractmethod".to_string()]), + ("override", vec!["override".to_string()]), + ("export", Vec::new()), + ] { + let source = format!( + "\ +class Box: + {modifier} def make(self) -> int: + return 7 +" + ); + let decorators = with_source(&source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{modifier}: {:?}", + module.declined + ); + module + .all_functions() + .find(|function| function.name.ends_with("make")) + .map(|function| dotted(&function.decorators)) + .unwrap_or_else(|| panic!("{modifier}: make is compiled")) + }); + assert_eq!(decorators, expected, "{modifier}"); + } +} + +#[test] +fn a_method_that_is_not_bound_to_its_receiver_carries_its_convention() { + // a method's first parameter used to be forced to the receiver whatever the + // decorators said, and these two say slot zero holds something else — so + // `Box.make(3)` compiled with `3` bound to a `Box` and raised at its first call. + // + // the convention rides on the method table entry now, so slot zero holds what + // python puts there: nothing at all for a static method, whose first written + // parameter keeps its own representation, and the *class* for a class method — an + // ordinary object, pointedly not an instance of the layout, so nothing derives a + // field read from it. + // + // and the decorator comes off the list: it is honoured by the emitted type rather + // than applied to it, and applying it as well would wrap the descriptor twice + for (source, binding, params) in [ + ( + "class Box:\n static def make(x: int) -> int:\n return x\n", + by_ir::function::Binding::Static, + vec![("x", RType::INT)], + ), + ( + "class Box:\n @staticmethod\n def make(x: int) -> int:\n return x\n", + by_ir::function::Binding::Static, + vec![("x", RType::INT)], + ), + ( + "class Box:\n @classmethod\n def make(cls, x: int) -> int:\n return x\n", + by_ir::function::Binding::Class, + vec![("cls", RType::OBJECT), ("x", RType::INT)], + ), + ( + "class Box:\n def make(self, x: int) -> int:\n return x\n", + by_ir::function::Binding::Instance, + vec![ + ( + "self", + RType::Instance { + class: "Box".to_string(), + exact: false, + }, + ), + ("x", RType::INT), + ], + ), + ] { + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{source}: {:?}", + module.declined + ); + let method = module + .all_functions() + .find(|function| function.name.ends_with("make")) + .unwrap_or_else(|| panic!("{source}: make is compiled")); + assert_eq!(method.binding, binding, "{source}"); + assert!(method.decorators.is_empty(), "{source}"); + let lowered: Vec<(&str, RType)> = method + .params() + .iter() + .map(|param| (param.name.as_deref().unwrap_or(""), param.ty.clone())) + .collect(); + let expected: Vec<(&str, RType)> = params + .iter() + .map(|(name, ty)| (*name, ty.clone())) + .collect(); + assert_eq!(lowered, expected, "{source}"); + }); + } } #[test] -fn a_module_defining_len_shadows_the_intrinsic() { - let ir = ir("\ -def len(x: int) -> int: - return x - -def f(a: int) -> int: - return len(a) -"); - assert!(ir.contains("call len(a)"), "{ir}"); +fn a_second_decorator_over_a_static_method_keeps_the_decline() { + // the runtime folds the remaining decorators onto the attribute it reads back off + // the finished type — and reading a static method back hands over the plain + // function it wraps, which would then be written back as an ordinary method. so + // the convention is only honoured natively where nothing else has to be applied + for source in [ + "class Box:\n @final\n @staticmethod\n def make() -> int:\n return 7\n", + "class Box:\n @staticmethod\n @final\n def make() -> int:\n return 7\n", + "class Box:\n @final\n @classmethod\n def make(cls) -> int:\n return 7\n", + ] { + let reasons = declines(source); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("a second decorator over")), + "{source}: {reasons:?}" + ); + } } #[test] -fn concatenating_two_strings_stays_a_string() { - let ir = ir("def f(a: str, b: str) -> str:\n return a + b\n"); - assert!(ir.contains("-> str"), "{ir}"); - assert!(ir.contains(" ++ "), "{ir}"); +fn a_convention_python_gives_a_method_itself_is_not_carried_twice() { + // python already makes each of these implicitly static or class, and an emitted + // generic class publishes a `__class_getitem__` of its own — so a table entry here + // would either duplicate the convention or collide with that entry + for source in [ + "class Box:\n @staticmethod\n def __new__(cls) -> object:\n return 7\n", + "class Box:\n @classmethod\n def __init_subclass__(cls) -> None:\n return None\n", + "class Box:\n @classmethod\n def __class_getitem__(cls, item: object) -> object:\n return item\n", + ] { + let reasons = declines(source); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("a convention of its own") + || reason.contains("fills a type slot")), + "{source}: {reasons:?}" + ); + } } #[test] -fn an_async_function_is_declined() { - assert!(decline("def f(a: int) -> None:\n try:\n pass\n except* ValueError:\n pass\n") - .contains("`except*`")); +fn a_global_a_frame_assigns_is_written_to_the_namespace_and_read_back_from_it() { + // a `global` declaration says where a name lives, and both halves of it have to + // agree: the write goes to the module namespace, and every read in the same frame + // comes back out of it. binding a register for either half is what made + // `mimetypes.init` set `inited` where nothing else could see it + for (source, function, writes, reads) in [ + ( + "seen = 0\n\ndef bump(n: int) -> int:\n global seen\n seen = n\n return seen\n", + "bump", + 1, + 1, + ), + // augmented assignment is a read and a write of the one place, so it is both + ( + "seen = 0\n\ndef bump(n: int) -> int:\n global seen\n seen += n\n return seen\n", + "bump", + 1, + 2, + ), + // a loop target is a binding like any other + ( + "seen = 0\n\ndef bump(ns: list[int]) -> int:\n global seen\n for seen in ns:\n pass\n return seen\n", + "bump", + 1, + 1, + ), + // a declaration with no assignment under it is redundant, and the name resolves + // where it already resolved + ( + "seen = 0\n\ndef read(n: int) -> int:\n global seen\n return seen + n\n", + "read", + 0, + 1, + ), + // the same name written by a frame that did *not* declare it is an ordinary + // local, and shadowing the global is what python does with it too + ( + "seen = 0\n\ndef shadow(n: int) -> int:\n seen = n\n return seen\n", + "shadow", + 0, + 0, + ), + ] { + let (stores, loads) = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{source}: {:?}", + module.declined + ); + let lowered = module + .functions + .iter() + .find(|candidate| candidate.name == function) + .unwrap_or_else(|| panic!("{function} was not emitted")); + let count = |wanted: fn(&Op) -> bool| { + lowered + .blocks + .iter() + .flat_map(|block| block.ops.iter()) + .filter(|op| wanted(op)) + .count() + }; + ( + count(|op| matches!(op, Op::StoreGlobal { name, .. } if name == "seen")), + count(|op| matches!(op, Op::LoadGlobal { name, .. } if name == "seen")), + ) + }); + assert_eq!((stores, loads), (writes, reads), "{source}"); + } } #[test] -fn a_decorated_function_still_compiles() { - // the decorator is applied at module init to the installed native function, - // so the body is compiled either way +fn a_global_a_nested_frame_declares_is_never_captured_from_the_frame_around_it() { + // the enclosing frame binds a local `seen` and the nested one declares `seen` + // global, so the two names are different places. the nested body only *reads* it, + // which is the case nothing else rules out: a body that wrote it would look like + // it owned the name anyway, so only the declaration says the enclosing local is + // the wrong place to capture let source = "\ -def deco(f: object) -> object: - return f +seen = 0 +tally = 0 -@deco -def f() -> None: - pass + +def outer(n: int) -> int: + seen = n + + def peek() -> int: + global seen + return seen + + return peek() + seen + + +def declared_out_here(n: int) -> int: + global tally + tally = n + + def look() -> int: + return tally + + return look() + tally "; - with_source(source, |db, env, model, suite| { + let (fields, loads) = with_source(source, |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); assert!(module.declined.is_empty(), "{:?}", module.declined); - let decorated = module - .functions + let nested: Vec<&by_ir::function::Function> = module + .classes .iter() - .find(|function| function.name == "f") - .expect("f is compiled"); - assert_eq!(decorated.decorators, vec!["deco".to_string()]); + .flat_map(|class| class.methods.iter()) + .filter(|candidate| candidate.name == "peek" || candidate.name == "look") + .collect(); + assert_eq!(nested.len(), 2, "both nested functions are emitted"); + ( + // neither name may become an environment field: there is nothing in the + // frame around either nested function for it to hold + module + .classes + .iter() + .flat_map(|class| class.fields.iter()) + .filter(|field| field.name == "seen" || field.name == "tally") + .count(), + nested + .iter() + .flat_map(|function| function.blocks.iter()) + .flat_map(|block| block.ops.iter()) + .filter(|op| { + matches!(op, Op::LoadGlobal { name, .. } if name == "seen" || name == "tally") + }) + .count(), + ) }); + assert_eq!((fields, loads), (0, 2)); } #[test] -fn a_computed_decorator_is_declined() { - // a call or an attribute would need its arguments evaluated at module init - let source = "\ -def make(n: int) -> object: - return n - -@make(1) -def f() -> None: - pass -"; - let reason = with_source(source, |db, env, model, suite| { - let module = crate::build_module(db, env, model, suite, "app", true); - module - .declined +fn a_static_method_that_suspends_declines() { + // a generator's state class is namespaced by the receiver's class, and neither of + // these has one — so two classes each with a static `values` would want a single + // state class between them + let reasons = + declines("class Box:\n @staticmethod\n def values() -> object:\n yield 1\n"); + assert!( + reasons .iter() - .find(|declined| declined.name == "f") - .map(|declined| declined.reason.clone()) - .unwrap_or_default() - }); - assert!(reason.contains("plain-name decorator"), "{reason}"); + .any(|(_, reason)| reason.contains("that suspends is not lowered yet")), + "{reasons:?}" + ); +} + +#[test] +fn a_static_method_and_a_function_of_one_name_get_environments_of_their_own() { + // a nested function lives on a generated environment class named after the frame + // that makes it, and a method's frame is namespaced by its class. a static method + // has no receiver to take that name from, so the name comes from the class the + // `def` was *written* in — otherwise these two ask for one class between them + with_source( + "\ +class Box: + @staticmethod + def add(n: int) -> int: + def inner(k: int) -> int: + return k + n + return inner(1) + + +def add(n: int) -> int: + def inner(k: int) -> int: + return k + n + 100 + return inner(1) +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let environments: Vec<&str> = module + .classes + .iter() + .map(|class| class.name.as_str()) + .filter(|name| name.ends_with("$env")) + .collect(); + assert_eq!(environments, vec!["Box$add$env", "add$env"]); + }, + ); } #[test] @@ -3063,7 +4003,7 @@ data class Point: .methods .iter() .find(|method| method.name == name) - .map(|method| method.decorators.clone()) + .map(|method| dotted(&method.decorators)) }; assert_eq!(decorators("total"), Some(vec!["property".to_string()])); assert_eq!(decorators("raw"), Some(vec!["doubling".to_string()])); @@ -4813,3 +5753,179 @@ class Held: ); assert_eq!(reasons, Vec::new()); } + +#[test] +fn a_method_defined_twice_in_a_class_body_is_declined() { + // two `def`s of one name bind whichever one ran, and they mangle to one C symbol — + // so a class with both emitted two `Box.value` entries and the module then failed to + // compile outright, which is worse than any wrong answer + let source = "\ +class Box: + def value(self) -> int: + return 1 + + def value(self) -> int: + return 2 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.classes.iter().all(|class| class.name != "Box"), + "Box must not be emitted" + ); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert!( + reasons + .iter() + .any(|reason| reason.contains("`value` is defined more than once")), + "{reasons:?}" + ); +} + +#[test] +fn a_module_level_function_defined_twice_is_declined() { + // the same in the module scope, which had nobody asking: three module-level `def _`s + // in `importlib/resources/_common.py` emitted one `by_m__` twice and the extension + // failed to build at all + let source = "\ +import functools + +@functools.cache +def _(n: int) -> int: + return n + +@functools.cache +def _(n: int) -> int: + return n + 1 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.functions.is_empty(), "nothing may compile"); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert_eq!(reasons.len(), 2, "{reasons:?}"); + assert!( + reasons + .iter() + .all(|reason| reason.contains("`_` is defined more than once")), + "{reasons:?}" + ); +} + +/// the twin's source keeps only the decorators module init will not re-apply +/// +/// a decorator init applies is evaluated there, over the compiled definition — so +/// leaving it on the twin's `def` evaluates it a second time and doubles whatever it did +/// on the way. a class's comes out for the same reason, because init applies that one to +/// the namespace entry. a *method's* stays: the class construction reads what it wrote — +/// `ABCMeta` computes `__abstractmethods__` from the namespace the body left — so taking +/// it out changes the class the twin builds rather than only when the decorator ran +#[test] +fn only_the_decorators_init_applies_come_out_of_the_twin() { + let source = "\ +def mark(f: object) -> object: + return f + + +@mark +def counted() -> int: + return 1 + + +@mark +class Held: + @mark + def value(self) -> int: + return 2 +"; + let twin = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + crate::without_init_decorators(source, &module).expect("the twin parses") + }); + assert_eq!(twin.matches("@mark").count(), 1, "{twin}"); + assert!(twin.contains(" @mark\n def value"), "{twin}"); + // the blanking keeps every line where it was, so a traceback through the twin still + // quotes the right one + assert_eq!(twin.lines().count(), source.lines().count(), "{twin}"); +} + +/// a decorator init does not re-apply stays on the twin's definition +/// +/// `staticmethod` is the shape: the method table honours the binding itself, so init has +/// nothing to apply and the twin's own `def` is the only thing that can +#[test] +fn a_decorator_init_does_not_re_apply_stays_on_the_twin() { + let source = "\ +class Held: + @staticmethod + def value() -> int: + return 2 +"; + let twin = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + crate::without_init_decorators(source, &module).expect("the twin parses") + }); + assert!(twin.contains("@staticmethod"), "{twin}"); +} + +/// a decorated definition the module reads cannot have its decorator moved to init +#[test] +fn a_decorated_definition_the_module_reads_declines() { + let reasons = declines( + "\ +def mark(f: object) -> object: + return f + + +@mark +def counted() -> int: + return 1 + + +at_import = counted() +", + ); + assert!( + reasons + .iter() + .any(|(name, reason)| name == "counted" && reason.contains("this module reads")), + "{reasons:?}" + ); +} + +/// and neither can a decorated class the module reads +#[test] +fn a_decorated_class_the_module_reads_declines() { + let reasons = declines( + "\ +def mark(c: object) -> object: + return c + + +@mark +class Held: + def value(self) -> int: + return 1 + + +table = [Held] +", + ); + assert!( + reasons + .iter() + .any(|(name, reason)| name == "Held" && reason.contains("this module reads")), + "{reasons:?}" + ); +} diff --git a/crates/by_opt/src/coalesce.rs b/crates/by_opt/src/coalesce.rs index 6c0c2abf1a..01bf1718ad 100644 --- a/crates/by_opt/src/coalesce.rs +++ b/crates/by_opt/src/coalesce.rs @@ -112,7 +112,7 @@ mod tests { fn module(function: by_ir::function::Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], classes: Vec::new(), declined: Vec::new(), diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 9b8928cdab..96cce8d981 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -272,6 +272,8 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::CallPython { dest, .. } | Op::CallValue { dest, .. } | Op::LoadGlobal { dest, .. } + | Op::StoreGlobal { dest, .. } + | Op::DeleteGlobal { dest, .. } | Op::LoadClass { dest, .. } | Op::ImportModule { dest, .. } | Op::ImportFrom { dest, .. } @@ -341,7 +343,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/dead_registers.rs b/crates/by_opt/src/dead_registers.rs index 48d0500e26..9c271c5a9b 100644 --- a/crates/by_opt/src/dead_registers.rs +++ b/crates/by_opt/src/dead_registers.rs @@ -168,6 +168,13 @@ fn rewrite_op( | Op::ImportModule { dest, .. } => { rewrite_dest(dest); } + Op::StoreGlobal { dest, value, .. } => { + rewrite_dest(dest); + rewrite_value(value, remap); + } + Op::DeleteGlobal { dest, .. } => { + rewrite_dest(dest); + } Op::ImportFrom { dest, module, .. } => { rewrite_dest(dest); rewrite_value(module, remap); @@ -447,7 +454,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index f68667e6ac..134f8545c1 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -520,7 +520,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index eb4efbbbda..d14e237e7c 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -150,6 +150,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::CallPython { .. } | Op::CallValue { .. } | Op::LoadGlobal { .. } + | Op::StoreGlobal { .. } + | Op::DeleteGlobal { .. } | Op::ImportModule { .. } | Op::ImportFrom { .. } | Op::NewInstance { .. } @@ -267,7 +269,7 @@ mod tests { fn module(functions: Vec) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions, declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index b4e6e58574..6c826c1119 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -150,7 +150,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(index))); let mut module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -185,7 +185,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(out))); let mut module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/refcount.rs b/crates/by_opt/src/refcount.rs index 4300c66cec..85b901dfa2 100644 --- a/crates/by_opt/src/refcount.rs +++ b/crates/by_opt/src/refcount.rs @@ -102,7 +102,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/str_append.rs b/crates/by_opt/src/str_append.rs index fa7b468894..77d1792b41 100644 --- a/crates/by_opt/src/str_append.rs +++ b/crates/by_opt/src/str_append.rs @@ -178,7 +178,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/str_item_compare.rs b/crates/by_opt/src/str_item_compare.rs index ff34080392..e6d2522f92 100644 --- a/crates/by_opt/src/str_item_compare.rs +++ b/crates/by_opt/src/str_item_compare.rs @@ -148,7 +148,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index 5c53df6efa..938fefa991 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -372,7 +372,7 @@ mod tests { fn module_with(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 4979a7e8bb..0b26ffa2e1 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -821,6 +821,29 @@ static inline char By_Truthy(PyObject *o) { #define By_TypeData(obj, cls) ((void *)(obj)) #endif +/* whether an emitted class may keep an instance dict beside its layout + * + * a managed dict lives in the pre-header, so it is the one form that leaves the struct, + * its base's prefix and every field offset alone — but walking and releasing it is + * `PyObject_VisitManagedDict` and `PyObject_ClearManagedDict`, which 3.13 published and + * nothing below it offers outside the internal headers. so a module holding such a class + * is left to its interpreted definitions on an older interpreter, decided at import + * rather than when the C is written */ +#if PY_VERSION_HEX >= 0x030D0000 +#define BY_HAS_MANAGED_DICT 1 +#define BY_MANAGED_DICT_FLAG Py_TPFLAGS_MANAGED_DICT +#define By_VisitManagedDict(obj, visit, arg) PyObject_VisitManagedDict((obj), (visit), (arg)) +#define By_ClearManagedDict(obj) PyObject_ClearManagedDict(obj) +#else +#define BY_HAS_MANAGED_DICT 0 +/* the flag is spelled in a static initializer, which is written whatever the interpreter + * — and below 3.11 there is no such flag to name at all. these three are never reached: + * the module falls back before any instance of such a class exists */ +#define BY_MANAGED_DICT_FLAG 0 +#define By_VisitManagedDict(obj, visit, arg) ((void)(obj), (void)(visit), (void)(arg)) +#define By_ClearManagedDict(obj) ((void)(obj)) +#endif + /* reading a local on a path that never assigned it. the phrasing is the running * python's, not the compiler's — it changed in 3.11 and a compiled module has to say * what the interpreter beside it would say */ @@ -892,6 +915,72 @@ static inline PyObject *By_LookupGlobalString(PyObject *dict, const char *name) return value; } +/* `root.a.b`: the root the way `LOAD_GLOBAL` resolves it, then a `getattr` each + * + * this is what a decorator expression written as a chain of attributes does, and all + * of what it does — every step is a read, which is why evaluating it at module init + * rather than where the `def` stood is faithful. a python identifier holds no `.`, so + * the path arrives as one string and is split back apart here */ +static inline PyObject *By_LookupDotted(PyObject *dict, const char *path) { + const char *dot = strchr(path, '.'); + PyObject *value; + if (dot == NULL) return By_LookupGlobalString(dict, path); + { + PyObject *key = By_InternedStr(path, (Py_ssize_t)(dot - path)); + if (key == NULL) return NULL; + value = By_LookupGlobal(dict, key); + Py_DECREF(key); + } + while (value != NULL && dot != NULL) { + const char *segment = dot + 1; + const char *next = strchr(segment, '.'); + Py_ssize_t length = next == NULL ? (Py_ssize_t)strlen(segment) + : (Py_ssize_t)(next - segment); + PyObject *attr = By_InternedStr(segment, length); + PyObject *got; + if (attr == NULL) { + Py_DECREF(value); + return NULL; + } + got = PyObject_GetAttr(value, attr); + Py_DECREF(attr); + Py_DECREF(value); + value = got; + dot = next; + } + return value; +} + +/* bind a name in the module namespace: an assignment under a `global` declaration + * + * this is the write `By_LookupGlobal` is the read of, and it has to reach the same + * dict. binding a register instead would keep the new value to the frame, where + * python's binding is the module's — every other reader sees it at once, the + * interpreted twin included, since that twin's `__globals__` *is* this dict. + * + * builtins are pointedly not consulted: python's `STORE_GLOBAL` binds in the module + * namespace whether or not the name already resolved to a builtin */ +static inline char By_StoreGlobal(PyObject *dict, PyObject *name, PyObject *value) { + if (dict == NULL || name == NULL || value == NULL) return 2; + return PyDict_SetItem(dict, name, value) < 0 ? 2 : 0; +} + +/* unbind a name in the module namespace: `del x` under a `global x` + * + * a dict raises `KeyError` for a key it does not hold and python raises `NameError` + * for a name it does not bind, so the one has to be translated into the other */ +static inline char By_DeleteGlobal(PyObject *dict, PyObject *name) { + if (dict == NULL || name == NULL) return 2; + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_NameError, "name '%U' is not defined", name); + } + return 2; + } + return 0; +} + /* whether a type spec can be built on this tuple of bases * * `PyType_FromSpecWithBases` gives the type it builds `type` as its own, so any base @@ -919,14 +1008,27 @@ static inline int By_SpecTakesBases(PyObject *bases) { * deallocation. * * a class statement works the shape out from every base at once, so where the offsets - * disagree with the layout base the interpreted definition is what answers */ -static inline int By_OffsetsHoldUp(PyTypeObject *type) { + * disagree with the layout base the interpreted definition is what answers. + * + * a spec that *asked* for a managed dict is the one exception, and the spec is passed in + * so that asking can be told from inheriting: python keeps a managed dict in a pre-header + * it allocates itself, so the room is there and the offset — the sentinel `-1` — is the + * answer that was wanted. without this a decorated class silently kept its interpreted + * definition while every compiled function went on reading that definition's instances as + * its own struct */ +static inline int By_OffsetsHoldUp(PyTypeObject *type, PyType_Spec *spec) { PyTypeObject *base = type->tp_base; if (base == NULL) { return 1; } - return type->tp_dictoffset == base->tp_dictoffset - && type->tp_weaklistoffset == base->tp_weaklistoffset; + if (type->tp_weaklistoffset != base->tp_weaklistoffset) { + return 0; + } + if (type->tp_dictoffset == base->tp_dictoffset) { + return 1; + } + return spec != NULL && (spec->flags & BY_MANAGED_DICT_FLAG) != 0 + && type->tp_dictoffset == -1; } /* the type for a class whose fields sit past a base's instance, or nothing at all @@ -987,7 +1089,74 @@ static inline PyObject *By_SpecClass(PyObject *module_dict, const char *name, PyErr_Clear(); return NULL; } - if (!By_OffsetsHoldUp((PyTypeObject *)cls)) { + if (!By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + +/* the type for a class appending storage past one *this module also appends to* + * + * `By_SpecClass` refuses a heap base outright, and a class this module writes is always + * one: the interpreted definition under that name is a `class` statement's type, whose + * `tp_dealloc` is `subtype_dealloc`. that refusal is the right answer for a base python + * built — `subtype_dealloc` picks the deallocator to chain to out of `Py_TYPE(self)`, + * finds this class's own, and calls it back until the stack runs out. + * + * a base *this module builds from a spec* is the one heap base that is not like that. + * its three slots are ones we emitted: each reads the base to chain to from the type + * that declared it, so the chain walks down to the outside base and stops. so the whole + * chain of appended storage can be built, innermost first, each spec standing on the + * finished type of the one below rather than on the interpreted definition. + * + * `base` is that finished type. what is checked here is that the interpreted definition + * agrees the two are related the way the emitted pair are: `base_name` is still the twin + * this module's `base` was built from — nothing of this module's own is installed yet — + * and the twin settled on exactly it, as its layout base and as its only base. anything + * else and the emitted type would answer a shape the source never wrote */ +static inline PyObject *By_SpecSubclass(PyObject *module_dict, const char *name, + PyType_Spec *spec, const char *base_name, + PyObject *base) { + PyObject *twin = By_LookupGlobalString(module_dict, name); + PyObject *twin_base; + PyObject *bases; + PyObject *cls; + int agrees; + if (twin == NULL) { + PyErr_Clear(); + return NULL; + } + twin_base = By_LookupGlobalString(module_dict, base_name); + if (twin_base == NULL) { + PyErr_Clear(); + Py_DECREF(twin); + return NULL; + } + agrees = PyType_Check(twin) && PyType_Check(twin_base) + && (PyObject *)((PyTypeObject *)twin)->tp_base == twin_base + && PyTuple_GET_SIZE(((PyTypeObject *)twin)->tp_bases) == 1 + && PyTuple_GET_ITEM(((PyTypeObject *)twin)->tp_bases, 0) == twin_base + && By_SpecTakesBases(((PyTypeObject *)twin)->tp_bases); + Py_DECREF(twin_base); + Py_DECREF(twin); + if (!agrees) { + return NULL; + } + bases = PyTuple_Pack(1, base); + if (bases == NULL) { + PyErr_Clear(); + return NULL; + } + cls = PyType_FromSpecWithBases(spec, bases); + Py_DECREF(bases); + if (cls == NULL) { + PyErr_Clear(); + return NULL; + } + /* the same last question `By_SpecClass` asks, and for the same reason: a spec adds + * neither a `__dict__` nor a weakref, so both offsets have to be the base's */ + if (!By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { Py_DECREF(cls); return NULL; } @@ -1126,6 +1295,120 @@ static inline int By_SetInNamespace(PyObject *ns, const char *key, PyObject *val return failed; } +/* what a value carried onto an emitted type becomes — defined with the rest of the twin + * machinery, and named here because a class namespace is written before that */ +static PyObject *By_TwinReplacement(PyObject *value, PyObject *const *twins, + PyObject *const *types, Py_ssize_t count); + +/* the class-level constants a class body wrote, and where their values come from + * + * the interpreted definition evaluated each of them once at class-definition time, and it + * is the only place the same object can come from — so the body that definition wrote is + * what they are read off, under the substitution every carried attribute takes. that body + * is captured while the fallback source runs and before any of the class's own decorators + * are handed it; `By_RunModuleBody` says why the finished class will not do. `twins` and + * `types` are the module's arrays and `classes` how many entries they hold; `body` is NULL + * for a class no interpreted `class` statement wrote, and then there is nothing to carry */ +typedef struct { + PyObject *body; + const char *const *names; + Py_ssize_t count; + PyObject *const *twins; + PyObject *const *types; + Py_ssize_t classes; +} By_ClassConstants; + +/* the value one of them takes, as a new reference + * + * NULL is "the body did not write that name", which is not a failure and leaves no + * exception set: a body under a conditional may not have written it */ +static inline PyObject *By_ConstantValue(const By_ClassConstants *constants, Py_ssize_t at) { + PyObject *value, *stands; + if (constants == NULL || constants->body == NULL) return NULL; + /* read out of the mapping rather than through a lookup on the class: a lookup runs + * the descriptor protocol, so a `__class_getitem__ = classmethod(f)` would come back + * as a method already bound to the interpreted class rather than as the classmethod + * the body wrote */ + value = PyDict_GetItemString(constants->body, constants->names[at]); + if (value == NULL) return NULL; + /* a value that only *reaches* a twin keeps what the body gave it, exactly as + * `By_CopyClassConstant` leaves it — this is the value half of that copy */ + stands = By_TwinReplacement(value, constants->twins, constants->types, constants->classes); + if (stands == NULL) stands = value; + return By_NewRef(stands); +} + +/* write the constants into a class namespace, and hand back what was written + * + * the mapping is `{name: value}` for exactly the names the captured body wrote, and it is + * what the class is checked against afterwards */ +static inline PyObject *By_CarryConstants(PyObject *ns, const By_ClassConstants *constants) { + PyObject *carried = PyDict_New(); + Py_ssize_t at; + if (carried == NULL) return NULL; + for (at = 0; constants != NULL && at < constants->count; at++) { + PyObject *value = By_ConstantValue(constants, at); + int failed; + if (value == NULL) continue; + failed = By_SetInNamespace(ns, constants->names[at], value) < 0 + || PyDict_SetItemString(carried, constants->names[at], value) < 0; + Py_DECREF(value); + if (failed) { + Py_DECREF(carried); + return NULL; + } + } + return carried; +} + +/* whether the finished class answers every constant with the object it was handed + * + * writing them into the namespace is what lets the metaclass see them, and it is enough + * for a metaclass that only *reads* one — a `__slots__` an `ABCMeta` passes to + * `type.__new__`, an `_fields` a registry records. it is not enough for one that *makes* + * something of what the body wrote: an `EnumType` handed `STRICT = 'strict'` builds a + * member out of it, and the member is not the value, so every reference the module body + * already took would name the old one. + * + * name for name is what separates the two, and it is asked of the class's own dict — + * which is where a `class` statement's namespace lands, entry for entry, and the only + * place the comparison can be made against the raw object the body wrote. a lookup on the + * class would run the descriptor protocol instead, so a `__class_getitem__ = classmethod(f)` + * would answer a freshly bound method and never be identical to anything. where the check + * fails, the interpreted definition stands — which is the answer such a class had before + * any of this. + * + * that fallback carries the limit every fallback in `By_BuildClass` carries: a twin + * extends the *twin's* base, so a class refused here while a base of this module's is + * emitted would answer `issubclass` False where python answers True. the compile-time + * cascade is what keeps that from arising and the runtime cannot reach as far — the + * choice left here is between the twin and a failed import, and the twin is the better + * of the two. nothing over the stdlib is refused here at all */ +static inline int By_ConstantsHeldUp(PyObject *cls, PyObject *carried) { + PyObject *name, *wanted; + PyObject *own = cls != NULL && PyType_Check(cls) ? ((PyTypeObject *)cls)->tp_dict : NULL; + Py_ssize_t position = 0; + if (carried == NULL) return 1; + while (PyDict_Next(carried, &position, &name, &wanted)) { + int same; + if (own != NULL) { + /* borrowed, and nothing here runs while the walk is open */ + same = PyDict_GetItem(own, name) == wanted; + } else { + /* a metaclass answering with something that is not a class at all */ + PyObject *got = PyObject_GetAttr(cls, name); + if (got == NULL) { + PyErr_Clear(); + return 0; + } + same = got == wanted; + Py_DECREF(got); + } + if (!same) return 0; + } + return 1; +} + /* one entry of a method table, as the descriptor a class namespace holds * * the three cases `type_add_methods` distinguishes: a class method and a static method @@ -1144,8 +1427,8 @@ static inline PyObject *By_MethodDescriptor(PyTypeObject *owner, PyMethodDef *de return PyDescr_NewMethod(owner, def); } -/* the class `meta(name, bases, namespace, **kwds)` builds, with `methods` in that - * namespace +/* the class `meta(name, bases, namespace, **kwds)` builds, with `methods` and + * `constants` in that namespace * * the methods go in *before* the call rather than onto the finished type, and both * halves of that matter. `type.__new__` fills the type slots from the namespace, so a @@ -1153,14 +1436,22 @@ static inline PyObject *By_MethodDescriptor(PyTypeObject *owner, PyMethodDef *de * reads the namespace — an `ABCMeta` deciding which of the base's abstract methods * this class left abstract — sees what the class actually defines. * + * the constants go in for the same reason, and that is the whole of what makes a class + * with one buildable this way: copied onto the *finished* type they would land behind the + * metaclass's back, and a `__slots__` that arrived after `type.__new__` had already given + * the instances a dict is not a `__slots__` at all. what the copy cannot promise, the + * check after the call does — see `By_ConstantsHeldUp` — and where the call does not get + * that far, the raise is the same refusal by another route. + * * the descriptors name `object` as their owner because the type they belong to is what * this call produces. that is also the more faithful answer: the interpreted twin holds * plain functions there, and a plain function checks no receiver either */ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const char *name, PyObject *bases, PyObject *orig_bases, - PyObject *kwds, PyMethodDef *methods) { + PyObject *kwds, PyMethodDef *methods, + const By_ClassConstants *constants) { PyMethodDef *def; - PyObject *module_name, *prepare, *args, *ns, *cls = NULL; + PyObject *module_name, *prepare, *args, *ns, *carried, *cls; PyObject *meta = By_Metaclass(bases, kwds); if (meta == NULL) return NULL; args = Py_BuildValue("(sO)", name, bases); @@ -1211,20 +1502,40 @@ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const cha return NULL; } } + /* after the methods, so that a body writing a name as both leaves the value there — + * which is the answer the check below is made against */ + carried = By_CarryConstants(ns, constants); + if (carried == NULL) { + Py_DECREF(ns); + Py_DECREF(meta); + return NULL; + } args = Py_BuildValue("(sOO)", name, bases, ns); Py_DECREF(ns); - if (args != NULL) { - cls = PyObject_Call(meta, args, kwds); - Py_DECREF(args); + if (args == NULL) { + Py_DECREF(carried); + Py_DECREF(meta); + return NULL; } + cls = PyObject_Call(meta, args, kwds); + Py_DECREF(args); Py_DECREF(meta); + /* the interpreted definition already built this class — the fallback source ran + * before any of this — so a metaclass raising here is the reconstruction being wrong + * rather than the class being unbuildable, and taking the whole import down for it + * would be the worst of the three answers. `ssl`'s `Purpose` is the case: `EnumType` + * is handed a namespace whose members are the twin's finished ones, and building a + * member out of a member raises before the check below could turn it down */ + if (cls == NULL) PyErr_Clear(); /* a `metaclass` that is not a type may hand back anything, and what it hands back is - * what the name means — but it is not a type this module can hang a constant or a - * decorated method on, so the interpreted definition is what stands under it */ - if (cls != NULL && !PyType_Check(cls)) { - Py_DECREF(cls); + * what the name means — but it is not a type this module can hang a decorated method + * on, so the interpreted definition is what stands under it. a class that disagrees + * with what its body wrote is turned down the same way and for the same reason */ + if (cls == NULL || !PyType_Check(cls) || !By_ConstantsHeldUp(cls, carried)) { + Py_XDECREF(cls); cls = By_LookupGlobalString(module_dict, name); } + Py_DECREF(carried); return cls; } @@ -1243,11 +1554,13 @@ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const cha * layout — so `through_metaclass` is false for a class with fields of its own, and the * interpreted definition the fallback already ran is what answers for it. it is false * again for anything the caller can only put on the *finished* type: a method decorator - * and a class-level constant both land after the metaclass has decided what the class - * defines, and a metaclass that reads its namespace would disagree with them */ + * lands after the metaclass has decided what the class defines, and a metaclass that + * reads its namespace would disagree with it. a class-level constant does not, because + * `constants` carries it into the namespace instead */ static inline PyObject *By_BuildClass(PyObject *module_dict, const char *name, PyObject *bases, PyObject *kwds, PyMethodDef *methods, - PyType_Spec *spec, int through_metaclass) { + PyType_Spec *spec, int through_metaclass, + const By_ClassConstants *constants) { PyObject *cls, *resolved; if (bases == NULL) return NULL; resolved = By_ResolveBases(bases); @@ -1257,13 +1570,14 @@ static inline PyObject *By_BuildClass(PyObject *module_dict, const char *name, } if (spec != NULL && By_SpecTakesBases(resolved)) { cls = PyType_FromSpecWithBases(spec, resolved); - if (cls != NULL && !By_OffsetsHoldUp((PyTypeObject *)cls)) { + if (cls != NULL && !By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { Py_DECREF(cls); cls = By_LookupGlobalString(module_dict, name); } } else if (through_metaclass) { cls = By_TypeThroughMetaclass(module_dict, name, resolved, - resolved == bases ? NULL : bases, kwds, methods); + resolved == bases ? NULL : bases, kwds, methods, + constants); } else { cls = By_LookupGlobalString(module_dict, name); } @@ -1473,28 +1787,6 @@ static inline char By_IsMatchSequence(PyObject *o) { } #endif -/* move a class-level constant from the interpreted definition onto the compiled - * type - * - * a *static* type is immutable to `setattr`, which is what licenses direct - * dispatch — so this writes the type's dict, the way a C extension declares its - * own class attributes - */ -static inline int By_CopyClassConstant(PyObject *module_dict, const char *class_name, - PyTypeObject *type, const char *name) { - PyObject *twin = PyDict_GetItemString(module_dict, class_name); - if (twin == NULL) return 0; - PyObject *value = PyObject_GetAttrString(twin, name); - if (value == NULL) { - PyErr_Clear(); - return 0; - } - int result = PyDict_SetItemString(type->tp_dict, name, value); - Py_DECREF(value); - if (result == 0) PyType_Modified(type); - return result; -} - /* the refusal `__annotations__` gives where a class's own could not be carried across * * `type_get_annotations` hands whatever it finds under that name in `tp_dict` to its own @@ -1873,6 +2165,213 @@ static inline int By_AdoptTwinAttributes(PyObject *const *twins, PyObject *const return 0; } +/* move a class-level constant from the interpreted definition onto the compiled type + * + * a *static* type is immutable to `setattr`, which is what licenses direct dispatch — so + * this writes the type's dict, the way a C extension declares its own class attributes. + * + * the value comes out of the body that definition wrote, so `attr = C` in a class body + * hands over the *interpreted* `C`, and copying that verbatim gives the type an attribute + * naming a class nothing else in the module can reach. a value that *is* a twin is + * therefore replaced by the type standing in for it, exactly as a carried attribute is. + * + * a value that merely *reaches* one is left as the interpreted definition had it, and that + * is the one place this differs from `By_AdoptTwinAttributes`. dropping it instead was + * built and backed out on the measurement: it loses 65 attributes over the corpus — + * `ipaddress` its network constants among them. absence would be the better failure if the + * reach were new, but it is a defect this copy has always had, and a question of its own + * rather than one to settle as a side effect of the identity + */ +static inline int By_CopyClassConstant(PyObject *body, PyTypeObject *type, const char *name, + PyObject *const *twins, PyObject *const *types, + Py_ssize_t count) { + const char *const names[] = {name}; + By_ClassConstants constants = {body, names, 1, twins, types, count}; + /* the same value a class built through its metaclass is handed before the call, so the + * two constructions cannot drift apart about what a constant is */ + PyObject *stands = By_ConstantValue(&constants, 0); + int result; + if (stands == NULL) return 0; + result = PyDict_SetItemString(type->tp_dict, name, stands); + Py_DECREF(stands); + if (result == 0) PyType_Modified(type); + return result; +} + +/* the module-level names still bound to an interpreted twin, moved onto what replaced it + * + * the whole module body runs against the interpreted definitions, so every name it binds + * to a class holds the twin — `Kind = C`, a re-export under another spelling, a name a + * conditional picked — while the compiled type only ever replaces the one name the + * `class` statement wrote. what that leaves is two classes of the same name in the same + * module: `Kind()` builds an object `isinstance(obj, C)` denies, and a compiled method + * handed one refuses it outright with `doesn't apply to a 'C' object`. + * + * a name that *is* a twin is the one shape that can be moved soundly, and it is the same + * substitution `By_TwinReplacement` makes for a carried attribute. it is made against + * whatever now stands under the class's own name rather than against the type directly, + * so a decorated class hands its aliases the decorator's answer — which is what the body + * bound them to — instead of the type the decorator was given. + * + * a value that merely *reaches* a twin is not moved and cannot be: an instance the body + * built has the twin for its type, and a list holding one is the same object the body + * kept. those stay as the body left them */ +static inline int By_RemapTwinAliases(PyObject *module_dict, PyObject *const *twins, + const char *const *names, Py_ssize_t count) { + /* the keys first: the dict is written while this walks, and only for keys it + * already holds, but nothing may run against it mid-walk either way */ + PyObject *keys = PyDict_Keys(module_dict); + Py_ssize_t at; + if (keys == NULL) return -1; + for (at = 0; at < PyList_GET_SIZE(keys); at++) { + PyObject *key = PyList_GET_ITEM(keys, at); + PyObject *value = PyDict_GetItem(module_dict, key); + Py_ssize_t index; + if (value == NULL) continue; + for (index = 0; index < count; index++) { + PyObject *stands; + if (twins[index] == NULL || value != twins[index]) continue; + stands = PyDict_GetItemString(module_dict, names[index]); + /* the class's own name already holds it, and one whose type was never + * installed still holds the twin — neither is a move */ + if (stands == NULL || stands == value) break; + if (PyDict_SetItem(module_dict, key, stands) < 0) { + Py_DECREF(keys); + return -1; + } + break; + } + } + Py_DECREF(keys); + return 0; +} + +/* `__build_class__`, recording what each module-level `class` statement wrote + * + * `state` is `(the real __build_class__, the mapping to record into)`. the class is built + * first and read afterwards, because the namespace itself is never handed back: python + * gives it to the metaclass and to nobody else. what `type.__new__` made of it is the + * closer thing anyway — it is exactly what the interpreted class holds at the moment + * before the first of its decorators is handed it */ +static PyObject *By_CaptureClassBody(PyObject *state, PyObject *args, PyObject *kwds) { + PyObject *cls = PyObject_Call(PyTuple_GET_ITEM(state, 0), args, kwds); + PyObject *name, *qualified, *body; + int outermost; + if (cls == NULL || PyTuple_GET_SIZE(args) < 2 || !PyType_Check(cls)) return cls; + if (((PyTypeObject *)cls)->tp_dict == NULL) return cls; + name = PyTuple_GET_ITEM(args, 1); + /* a class written inside a function can be named the same as one at module level and + * is not the same class. `f..C` against `C` is what tells them apart, and the + * body function python passes here is what carries that qualified name */ + qualified = PyObject_GetAttrString(PyTuple_GET_ITEM(args, 0), "__qualname__"); + if (qualified == NULL) { + PyErr_Clear(); + return cls; + } + outermost = PyObject_RichCompareBool(qualified, name, Py_EQ); + Py_DECREF(qualified); + if (outermost != 1) { + if (outermost < 0) PyErr_Clear(); + return cls; + } + body = PyDict_Copy(((PyTypeObject *)cls)->tp_dict); + /* a body that cannot be recorded is raised out of the `class` statement rather than + * passed over: what would follow is a type carrying no constants at all, and for a + * decorated class that is the defect this capture exists to remove */ + if (body == NULL || PyDict_SetItem(PyTuple_GET_ITEM(state, 1), name, body) < 0) { + Py_XDECREF(body); + Py_DECREF(cls); + return NULL; + } + Py_DECREF(body); + return cls; +} + +/* run a module's fallback source, capturing each class body before its decorators run + * + * the source is the whole interpreted module and it runs to completion before any emitted + * type is built, so by the time a class-level constant is copied onto one, the class it + * would be copied off has already been through its own decorators. a decorator that only + * *reads* the class leaves the value where the body put it, but one that makes something + * of it does not: `@dataclass` deletes the `field(init=False)` a body wrote, and leaves a + * bare `2` where `field(default=2, repr=False)` stood. + * + * so the body is taken while it still is the body. python routes every `class` statement + * through `__build_class__`, and the one a statement reaches is the `__build_class__` of + * *its own frame's* builtins — so a copy of the builtins mapping, put in this module's + * dict, reaches this module's body and nothing else in the process. swapping the entry in + * the real builtins instead would be seen by every thread importing at the same time, + * which on a free-threaded interpreter is a live hazard rather than a theoretical one. + * + * the copy outlives the exec whatever is done with it: python gives a function the + * builtins its defining frame had, so every function this body defines holds this dict for + * as long as it lives. that is why the real entry is put back afterwards rather than the + * dict simply dropped — otherwise a class one of those functions made, at any later point + * in the process, would still be recorded here. + * + * hands back `{name: body}` for the classes the body wrote at module level, as a new + * reference, or NULL with an exception set where the body raised */ +static inline PyObject *By_RunModuleBody(const char *source, PyObject *dict) { + static PyMethodDef capture = {"__build_class__", + (PyCFunction)(void (*)(void))By_CaptureClassBody, + METH_VARARGS | METH_KEYWORDS, NULL}; + PyObject *bodies, *stood, *mapping, *builtins, *real, *state, *wrapper, *result; + int failed; + bodies = PyDict_New(); + if (bodies == NULL) return NULL; + /* an emitted module's dict has no `__builtins__` of its own, and python would then + * give the body's frame the running interpreter's */ + stood = PyDict_GetItemString(dict, "__builtins__"); + if (stood == NULL) stood = PyEval_GetBuiltins(); + Py_XINCREF(stood); + mapping = stood != NULL && PyModule_Check(stood) ? PyModule_GetDict(stood) : stood; + builtins = mapping != NULL && PyDict_Check(mapping) ? PyDict_Copy(mapping) : NULL; + real = builtins == NULL ? NULL : PyDict_GetItemString(builtins, "__build_class__"); + if (real == NULL) { + Py_XDECREF(builtins); + Py_XDECREF(stood); + Py_DECREF(bodies); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, + "no builtins `__build_class__` to run the module body against"); + } + return NULL; + } + Py_INCREF(real); + state = PyTuple_Pack(2, real, bodies); + wrapper = state == NULL ? NULL : PyCFunction_New(&capture, state); + Py_XDECREF(state); + failed = wrapper == NULL || PyDict_SetItemString(builtins, "__build_class__", wrapper) < 0 + || PyDict_SetItemString(dict, "__builtins__", builtins) < 0; + Py_XDECREF(wrapper); + result = failed ? NULL : PyRun_String(source, Py_file_input, dict, dict); + Py_XDECREF(result); + /* whatever the body did, the capture stops here */ + { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + if (PyDict_SetItemString(builtins, "__build_class__", real) < 0 + || PyDict_SetItemString(dict, "__builtins__", stood) < 0) { + PyErr_Clear(); + } + PyErr_Restore(type, value, traceback); + } + Py_DECREF(real); + Py_DECREF(builtins); + Py_DECREF(stood); + if (failed || result == NULL) { + Py_DECREF(bodies); + return NULL; + } + return bodies; +} + +/* the body captured for one class, as a borrowed reference, or NULL where there is none */ +static inline PyObject *By_ClassBody(PyObject *bodies, const char *name) { + if (bodies == NULL) return NULL; + return PyDict_GetItemString(bodies, name); +} + /* the answer a class pattern gives when the attribute it named is simply absent * * a missing attribute is *no match*, not an error — `case Point(z=1):` against a @@ -2837,10 +3336,10 @@ static inline PyObject *By_Method(PyObject *fn) { /* apply a method's decorators, innermost first, to the finished type — which is the * only place a type spec leaves for them. * - * each decorator is looked up in the module namespace, so `@property` and a - * user-defined one resolve the same way. they are folded in memory and the result - * written once, which is also what a class body does: the namespace never holds a - * half-decorated method. `PyType_Modified` is what makes the change visible — the + * each decorator is resolved out of the module namespace by `By_LookupDotted`, so + * `@property` and `@abc.abstractmethod` come out the same way. they are folded in + * memory and the result written once, which is also what a class body does: the + * namespace never holds a half-decorated method. `PyType_Modified` is what makes the change visible — the * attribute cache would otherwise keep serving the undecorated one */ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, const char *owner, const char *name, @@ -2863,7 +3362,7 @@ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, } for (index = count; index > 0; index--) { PyObject *args[1] = {target}; - PyObject *fn = By_LookupGlobalString(dict, decorators[index - 1]); + PyObject *fn = By_LookupDotted(dict, decorators[index - 1]); PyObject *wrapped; if (fn == NULL) { Py_DECREF(target); @@ -2885,9 +3384,10 @@ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, return 0; } -/* apply `dict[decorator]` to `dict[name]`, in place. this is what lets a - * decorated function still be compiled: the native one goes into the namespace, - * then the decorator wraps it, exactly as the `def` statement would have */ +/* apply the decorator `decorator` names to `dict[name]`, in place. this is what + * lets a decorated function still be compiled: the native one goes into the + * namespace, then the decorator wraps it, exactly as the `def` statement would + * have. `decorator` is a dotted path — see `By_LookupDotted` */ static inline int By_ApplyDecorator(PyObject *dict, const char *name, const char *decorator) { PyObject *target = PyDict_GetItemString(dict, name); if (target == NULL) { @@ -2895,7 +3395,7 @@ static inline int By_ApplyDecorator(PyObject *dict, const char *name, const char return -1; } Py_INCREF(target); - PyObject *fn = By_LookupGlobalString(dict, decorator); + PyObject *fn = By_LookupDotted(dict, decorator); if (fn == NULL) { Py_DECREF(target); return -1; @@ -3197,13 +3697,6 @@ static inline PyObject *By_PackInitKwargs(PyObject *kwds, const char *const *nam return packed; } -/* report every parameter the caller left out, in cpython's own wording. - * - * matching the message matters more than it looks: the differential harness compares - * exception text, and a difference there is a difference a user would see */ -/* every parameter with no default that nothing filled, named the way python names - * them — and positional and keyword-only are counted separately, because python - * reports them in two different sentences */ /* python began qualifying a method by its class in 3.10, so the name the compiler * wrote is trimmed back to its tail on an interpreter that would not have used it */ static inline const char *By_ErrorName(const char *fname) { @@ -3215,6 +3708,16 @@ static inline const char *By_ErrorName(const char *fname) { #endif } +/* every parameter with no default that nothing filled, named the way python names them + * — positional and keyword-only counted separately, because python reports them in two + * different sentences + * + * this is the wording of *last resort*: [`By_Rephrase`] runs first and lets the + * interpreter word the refusal itself, and only a shape it could not build falls back + * to here. so the list joined below is deliberately left as it always was, one comma + * short of python's — a differential test that sees this text is a test whose rephrasing + * never ran, which is the one thing a comparison of two identical strings could not + * otherwise tell anyone */ static inline int By_CheckRequired(const char *const *names, const unsigned char *required, Py_ssize_t count, Py_ssize_t kwonly, PyObject **out, const char *fname) { @@ -3255,30 +3758,162 @@ static inline int By_CheckRequired(const char *const *names, const unsigned char return 0; } +/* a spelling no parameter already has, for a synthetic one that has to be named + * + * the receiver, the `*args` and the `**kwargs` [`By_Rephrase`] writes are named in + * source nothing reads back, so any free spelling does and underscores are appended + * until one is free. free of the *real* names is not on its own enough, though: + * python offers a near miss to a caller who spelled a keyword wrongly, and it draws + * that suggestion from the parameters between the positional-only run and the end of + * the keyword-only one. a `*args` or `**kwargs` name lies outside that range and a + * positional-only one before it, which is why the receiver is written as one */ +static inline void By_SpareName(char *buffer, size_t size, const char *stem, + const char *const *names, Py_ssize_t count) { + size_t used = strlen(stem); + if (used + 1 > size) used = size - 1; + memcpy(buffer, stem, used); + buffer[used] = '\0'; + while (used + 1 < size) { + Py_ssize_t i = 0; + while (i < count && strcmp(buffer, names[i]) != 0) i++; + if (i == count) return; + buffer[used++] = '_'; + buffer[used] = '\0'; + } +} + +/* the caller's positionals with the receiver python counts put back in front of them */ +static inline PyObject *By_ShapeArgs(PyObject *args, Py_ssize_t receiver) { + Py_ssize_t nargs = args == NULL ? 0 : PyTuple_GET_SIZE(args); + Py_ssize_t extra = receiver ? 1 : 0; + PyObject *made = PyTuple_New(nargs + extra); + if (made == NULL) return NULL; + if (extra) PyTuple_SET_ITEM(made, 0, By_NewRef(Py_None)); + for (Py_ssize_t i = 0; i < nargs; i++) { + PyTuple_SET_ITEM(made, i + extra, By_NewRef(PyTuple_GET_ITEM(args, i))); + } + return made; +} + +/* the refusal the interpreter itself would word for a call to a function of this shape + * + * nothing in the c api formats one. `format_missing`, `too_many_positional` and + * `format_kwargs_error` are all static to `ceval.c`, and their wording is fussier than + * it looks: `and` from two names up, a comma *before* that `and` from three up, a range + * rather than a count once any parameter has a default, and a receiver counted in the + * arity sentence but not in the missing-argument one. writing those rules out is what + * left the comma out of this message for the whole of the project's life, and it was + * right when it was written — so the next rule to change would go the same way + * + * so rather than the rules, the *shape*: a python function with the same parameters, + * handed the same call. its body is `pass`, so the only thing the call can do is raise + * what the interpreter raises for the real one. returns 1 having left that exception + * pending, or 0 having left none — which is the two binders disagreeing, and is why the + * caller's own wording stays behind this + * + * the caller's exception must be off the thread before this is reached: it compiles and + * it calls, and neither is reached with one pending */ +static inline int By_Rephrase(const char *const *names, const unsigned char *required, + Py_ssize_t count, Py_ssize_t posonly, Py_ssize_t kwonly, + int variadic, int extras, const char *fname, + Py_ssize_t receiver, PyObject *args, PyObject *kwds) { + char self_name[32], rest_name[32], keys_name[32]; + By_SpareName(self_name, sizeof(self_name), "_by_self", names, count); + By_SpareName(rest_name, sizeof(rest_name), "_by_rest", names, count); + By_SpareName(keys_name, sizeof(keys_name), "_by_keys", names, count); + /* a keyword spelled like the synthetic receiver would bind to it, and the shape + * would then answer about a parameter the real function does not have */ + if (kwds != NULL && receiver && PyDict_GetItemString(kwds, self_name) != NULL) return 0; + + Py_ssize_t limit = count - kwonly; + PyObject *source = PyUnicode_FromString("def _("); + if (receiver) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("%s, ", self_name)); + /* positional-only, so that no near miss is ever offered against it. where the + * function has a positional-only run of its own the marker comes after that */ + if (posonly == 0) PyUnicode_AppendAndDel(&source, PyUnicode_FromString("/, ")); + } + for (Py_ssize_t i = 0; i < limit; i++) { + PyUnicode_AppendAndDel( + &source, PyUnicode_FromFormat("%s%s, ", names[i], required[i] ? "" : "=None")); + if (i + 1 == posonly) PyUnicode_AppendAndDel(&source, PyUnicode_FromString("/, ")); + } + if (variadic) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("*%s, ", rest_name)); + } else if (kwonly > 0) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromString("*, ")); + } + for (Py_ssize_t i = limit; i < count; i++) { + PyUnicode_AppendAndDel( + &source, PyUnicode_FromFormat("%s%s, ", names[i], required[i] ? "" : "=None")); + } + if (extras) PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("**%s", keys_name)); + PyUnicode_AppendAndDel(&source, PyUnicode_FromString("): pass\n")); + + /* the module namespace the definition lands in, which is also where it is read back + * from. nothing in the source needs a builtin, and evaluation supplies the + * interpreter's own where a namespace carries none */ + PyObject *shape = NULL, *scope = NULL; + const char *text = source == NULL ? NULL : PyUnicode_AsUTF8(source); + if (text != NULL) scope = PyDict_New(); + if (scope != NULL) { + PyObject *ran = PyRun_String(text, Py_file_input, scope, scope); + Py_XDECREF(ran); + if (ran != NULL) shape = PyDict_GetItemString(scope, "_"); + } + int reworded = 0; + if (shape != NULL) { + /* every one of these messages names the *qualified* name, which for a function + * is the one it carries rather than the one its code object was compiled with */ + PyObject *label = PyUnicode_FromString(fname); + int named = label != NULL && PyObject_SetAttrString(shape, "__qualname__", label) == 0; + Py_XDECREF(label); + PyObject *positional = named ? By_ShapeArgs(args, receiver) : NULL; + if (positional != NULL) { + PyObject *answer = PyObject_Call(shape, positional, kwds); + Py_DECREF(positional); + if (answer != NULL) Py_DECREF(answer); + else if (PyErr_ExceptionMatches(PyExc_TypeError)) reworded = 1; + } + } + Py_XDECREF(source); + Py_XDECREF(scope); + /* a shape that could not be built, or one that refused for a reason of its own, + * leaves the thread as it found it */ + if (!reworded) PyErr_Clear(); + return reworded; +} + /* the constructor's binding: the same rules [`By_BindArgs`] applies, read off a tuple * and a dict rather than a fastcall vector — which is the whole of what differs * * `out[i]` receives a *borrowed* pointer, or NULL where the caller supplied nothing * and the default fills it. python counts `self` in its arity message and not in its * missing-argument one, so this does too */ -static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const *names, - Py_ssize_t count, const unsigned char *required, - Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, - int variadic, int extras, const char *fname, int inherited) { +static inline int By_BindInitPlain(PyObject *args, PyObject *kwds, + const char *const *names, Py_ssize_t count, + const unsigned char *required, Py_ssize_t posonly, + Py_ssize_t kwonly, PyObject **out, int variadic, + int extras, const char *fname, int inherited) { fname = By_ErrorName(fname); for (Py_ssize_t i = 0; i < count; i++) out[i] = NULL; Py_ssize_t nargs = args == NULL ? 0 : PyTuple_GET_SIZE(args); /* a keyword-only parameter is one nothing positional can reach, so the run a * caller may fill positionally ends where they begin */ Py_ssize_t positional_limit = count - kwonly; - if (nargs > positional_limit && !variadic) { - /* a class with no `__init__` at all is rejected by `object.__init__`, which - * names the class and does not count a receiver it never had. a *written* - * `def __init__(self)` takes no arguments either and still reports as a method */ - if (inherited) { + /* a class with no `__init__` at all is rejected by `object.__init__`, which names + * the class, does not count a receiver it never had, and asks only whether it was + * given anything — a keyword is as much an excess argument as a positional, and + * saying which one would be a distinction `object_init` never draws. such a class + * takes no parameters at all, so there is nothing else the call could be about. a + * *written* `def __init__(self)` takes no arguments either and still reports as a + * method, which is why this turns on how the class was written and not on `count` */ + if (inherited) { + if (nargs > 0 || (kwds != NULL && PyDict_Size(kwds) > 0)) { PyErr_Format(PyExc_TypeError, "%s() takes no arguments", fname); return -1; } + } else if (nargs > positional_limit && !variadic) { return By_TooManyPositional(fname, required, positional_limit, nargs, 1); } Py_ssize_t positional = nargs < positional_limit ? nargs : positional_limit; @@ -3306,6 +3941,32 @@ static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const return By_CheckRequired(names, required, count, kwonly, out, fname); } +/* the same binding, with a refusal put back into the interpreter's own words + * + * the plain binding writes nothing but `out`, and rewrites all of it before reading any + * — so where the shape declines to reword, running it a second time is how its own + * message comes back, and nothing has to be carried across the attempt */ +static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const *names, + Py_ssize_t count, const unsigned char *required, + Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, + int variadic, int extras, const char *fname, int inherited) { + if (By_BindInitPlain(args, kwds, names, count, required, posonly, kwonly, out, variadic, + extras, fname, inherited) == 0) { + return 0; + } + /* a class that wrote no `__init__` is refused by `object.__init__`, which is not a + * python function and has no shape to model. and a refusal that is not a `TypeError` + * is not an arity one — it is the binding itself having failed */ + if (inherited || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; + PyErr_Clear(); + if (By_Rephrase(names, required, count, posonly, kwonly, variadic, extras, + By_ErrorName(fname), 1, args, kwds)) { + return -1; + } + return By_BindInitPlain(args, kwds, names, count, required, posonly, kwonly, out, + variadic, extras, fname, inherited); +} + /* bind fastcall arguments to parameter positions, honouring keywords. * * `receiver` is 1 for a method, whose `self` arrives outside the vector but which @@ -3314,11 +3975,12 @@ static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const * `out[i]` receives a *borrowed* pointer, or NULL where the caller did not supply * that parameter — the wrapper fills those from the defaults. returns -1 with an * exception set on a duplicate, an unexpected name, or too many positionals */ -static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, - const char *const *names, Py_ssize_t count, - const unsigned char *required, Py_ssize_t posonly, - Py_ssize_t kwonly, PyObject **out, int variadic, int extras, - const char *fname, Py_ssize_t receiver) { +static inline int By_BindArgsPlain(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames, const char *const *names, + Py_ssize_t count, const unsigned char *required, + Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, + int variadic, int extras, const char *fname, + Py_ssize_t receiver) { fname = By_ErrorName(fname); /* a keyword-only parameter is one nothing positional can reach, so the run a * caller may fill positionally ends where they begin */ @@ -3351,6 +4013,54 @@ static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject return By_CheckRequired(names, required, count, kwonly, out, fname); } +/* a fastcall vector as the tuple and dict a plain call takes. only the error path needs + * either, and it is cold */ +static inline PyObject *By_VectorTuple(PyObject *const *args, Py_ssize_t nargs) { + PyObject *made = PyTuple_New(nargs); + if (made == NULL) return NULL; + for (Py_ssize_t i = 0; i < nargs; i++) PyTuple_SET_ITEM(made, i, By_NewRef(args[i])); + return made; +} + +static inline PyObject *By_VectorKwds(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) { + PyObject *made = PyDict_New(); + if (made == NULL || kwnames == NULL) return made; + Py_ssize_t keywords = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t k = 0; k < keywords; k++) { + if (PyDict_SetItem(made, PyTuple_GET_ITEM(kwnames, k), args[nargs + k]) < 0) { + Py_DECREF(made); + return NULL; + } + } + return made; +} + +/* the same binding, with a refusal put back into the interpreter's own words */ +static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + const char *const *names, Py_ssize_t count, + const unsigned char *required, Py_ssize_t posonly, + Py_ssize_t kwonly, PyObject **out, int variadic, int extras, + const char *fname, Py_ssize_t receiver) { + if (By_BindArgsPlain(args, nargs, kwnames, names, count, required, posonly, kwonly, out, + variadic, extras, fname, receiver) == 0) { + return 0; + } + if (!PyErr_ExceptionMatches(PyExc_TypeError)) return -1; + PyErr_Clear(); + PyObject *tuple = By_VectorTuple(args, nargs); + PyObject *dict = tuple == NULL ? NULL : By_VectorKwds(args, nargs, kwnames); + int reworded = dict != NULL + && By_Rephrase(names, required, count, posonly, kwonly, variadic, extras, + By_ErrorName(fname), receiver, tuple, dict); + Py_XDECREF(tuple); + Py_XDECREF(dict); + if (reworded) return -1; + PyErr_Clear(); + return By_BindArgsPlain(args, nargs, kwnames, names, count, required, posonly, kwonly, + out, variadic, extras, fname, receiver); +} + /* `with EXPR`: the manager's `__enter__`, looked up on the *type* the way the * interpreter does rather than on the instance */ /* `__aenter__` and `__aexit__`, which hand back *awaitables* rather than answers diff --git a/crates/by_rt/src/lib.rs b/crates/by_rt/src/lib.rs index 4bad569f38..32210b1036 100644 --- a/crates/by_rt/src/lib.rs +++ b/crates/by_rt/src/lib.rs @@ -38,8 +38,39 @@ mod tests { "By_ObjCompare", "By_Truthy", "By_ApplyMethodDecorators", + "By_SpecClass", + "By_SpecSubclass", ] { assert!(BY_H.contains(symbol), "the runtime is missing {symbol}"); } } + + /// both constructions for a class whose fields sit past a base's instance answer + /// with nothing rather than with something else, because the caller's only move is + /// to leave the whole module as its interpreted definition already built it. one + /// that raised instead would make module init fail the import + #[test] + fn a_layout_that_does_not_hold_up_is_refused_rather_than_raised() { + for construction in ["By_SpecClass", "By_SpecSubclass"] { + let body = BY_H + .split_once(&format!("static inline PyObject *{construction}(")) + .expect("the construction is defined") + .1 + .split_once("\n}\n") + .expect("it ends") + .0; + assert!( + body.contains("if (!By_OffsetsHoldUp((PyTypeObject *)cls"), + "{construction} checks the finished type's offsets" + ); + assert!( + !body.contains("PyErr_Set") && !body.contains("PyErr_Format"), + "{construction} raises nothing: {body}" + ); + assert!( + body.contains("return NULL;"), + "{construction} answers with nothing" + ); + } + } } diff --git a/crates/by_transforms/src/reverse_transforms/callable.rs b/crates/by_transforms/src/reverse_transforms/callable.rs index cdc98a7268..d5cccd8977 100644 --- a/crates/by_transforms/src/reverse_transforms/callable.rs +++ b/crates/by_transforms/src/reverse_transforms/callable.rs @@ -17,7 +17,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct CallableReverse<'src> { types: &'src dyn TypeInfo, @@ -55,24 +55,11 @@ impl<'src> CallableReverse<'src> { } fn is_callable_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Callable" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Callable" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Callable") && self.types.subscript_is_type_context(expr) } fn is_type_context_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// rewrite the punctuation of `Callable[, ]` into the arrow diff --git a/crates/by_transforms/src/reverse_transforms/intersection.rs b/crates/by_transforms/src/reverse_transforms/intersection.rs index 7cebaba943..5aa516316f 100644 --- a/crates/by_transforms/src/reverse_transforms/intersection.rs +++ b/crates/by_transforms/src/reverse_transforms/intersection.rs @@ -9,7 +9,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct IntersectionReverse<'src> { source: &'src str, @@ -31,16 +31,7 @@ impl<'src> IntersectionReverse<'src> { } fn is_intersection_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => { - n.id.as_str() == "Intersection" && self.types.subscript_is_type_context(n) - } - Expr::Attribute(a) => { - a.attr.id.as_str() == "Intersection" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Intersection") && self.types.subscript_is_type_context(expr) } fn rewrite(&mut self, expr: &Expr) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/literal_types.rs b/crates/by_transforms/src/reverse_transforms/literal_types.rs index c67bbe8399..2465ca1ccf 100644 --- a/crates/by_transforms/src/reverse_transforms/literal_types.rs +++ b/crates/by_transforms/src/reverse_transforms/literal_types.rs @@ -14,7 +14,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, ExprSubscript, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralReverse<'src> { source: &'src str, @@ -38,17 +38,7 @@ impl<'src> LiteralReverse<'src> { /// `Literal` or `typing.Literal` / `typing_extensions.Literal`, where the /// bare name resolves to an import or is unresolved fn is_literal_name(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => n.id.as_str() == "Literal" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Literal" - && match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - } - } - _ => false, - } + trailing_name(value) == Some("Literal") && self.types.subscript_is_type_context(value) } fn rewrite_literal_subscript(&self, s: &ExprSubscript) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/not_type.rs b/crates/by_transforms/src/reverse_transforms/not_type.rs index f451c34141..233515a989 100644 --- a/crates/by_transforms/src/reverse_transforms/not_type.rs +++ b/crates/by_transforms/src/reverse_transforms/not_type.rs @@ -8,7 +8,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Operator, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct NotTypeReverse<'src> { source: &'src str, @@ -30,14 +30,7 @@ impl<'src> NotTypeReverse<'src> { } fn is_not_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Not" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Not" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Not") && self.types.subscript_is_type_context(expr) } fn rewrite(&mut self, expr: &Expr) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/subscript.rs b/crates/by_transforms/src/reverse_transforms/subscript.rs index 9109a0642a..63246e1ff7 100644 --- a/crates/by_transforms/src/reverse_transforms/subscript.rs +++ b/crates/by_transforms/src/reverse_transforms/subscript.rs @@ -15,7 +15,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct SubscriptReverse<'src> { source: &'src str, @@ -33,14 +33,7 @@ impl<'src> SubscriptReverse<'src> { } fn is_type_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } } diff --git a/crates/by_transforms/src/reverse_transforms/tuple_type.rs b/crates/by_transforms/src/reverse_transforms/tuple_type.rs index c10978134b..b86b4f3046 100644 --- a/crates/by_transforms/src/reverse_transforms/tuple_type.rs +++ b/crates/by_transforms/src/reverse_transforms/tuple_type.rs @@ -16,7 +16,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct TupleTypeReverse<'src> { source: &'src str, @@ -58,19 +58,15 @@ impl<'src> TupleTypeReverse<'src> { fn is_tuple_name(&self, expr: &Expr) -> bool { match expr { - Expr::Name(n) => n.id.as_str() == "tuple" && self.types.subscript_is_type_context(n), + Expr::Name(_) => { + trailing_name(expr) == Some("tuple") && self.types.subscript_is_type_context(expr) + } _ => false, } } fn is_type_context_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// `*tuple[T, ...]` as a tuple element round-trips to the basedpython diff --git a/crates/by_transforms/src/reverse_transforms/type_is.rs b/crates/by_transforms/src/reverse_transforms/type_is.rs index 58bf1de721..6eccbc1829 100644 --- a/crates/by_transforms/src/reverse_transforms/type_is.rs +++ b/crates/by_transforms/src/reverse_transforms/type_is.rs @@ -10,7 +10,7 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct TypeIsReverse<'src> { source: &'src str, @@ -32,14 +32,7 @@ impl<'src> TypeIsReverse<'src> { } fn is_type_is(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "TypeIs" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "TypeIs" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("TypeIs") && self.types.subscript_is_type_context(expr) } } diff --git a/crates/by_transforms/src/reverse_transforms/unpack.rs b/crates/by_transforms/src/reverse_transforms/unpack.rs index 0bc0319305..a45b79247d 100644 --- a/crates/by_transforms/src/reverse_transforms/unpack.rs +++ b/crates/by_transforms/src/reverse_transforms/unpack.rs @@ -11,7 +11,7 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct UnpackReverse<'src> { source: &'src str, @@ -29,14 +29,7 @@ impl<'src> UnpackReverse<'src> { } fn is_unpack_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Unpack" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Unpack" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Unpack") && self.types.subscript_is_type_context(expr) } fn process_vararg_annotation(&mut self, ann: &Expr) { diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index a7d0f965d6..160bbe4074 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -17,7 +17,7 @@ use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping, }; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralType<'src> { source: &'src str, @@ -43,26 +43,12 @@ impl<'src> LiteralType<'src> { /// Whether a `Subscript.value` resolves to something whose subscript slice /// is a type-argument position. fn is_type_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// Is `value` a reference to the named typing special form? fn is_typing_name(&self, value: &Expr, name: &str) -> bool { - match value { - Expr::Name(n) => n.id.as_str() == name && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == name - && matches!(a.value.as_ref(), Expr::Name(base) if self.types.attr_base_is_type_context(base)) - } - _ => false, - } + trailing_name(value) == Some(name) && self.types.subscript_is_type_context(value) } fn is_annotated_name(&self, value: &Expr) -> bool { diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index d03022ea35..4d35e7ad13 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -566,6 +566,57 @@ mod tests { check("x = 1 + 1\n", "x = 1 + 1\n"); } + #[test] + fn a_module_attribute_that_is_not_a_type_keeps_its_subscript_key() { + // `sys.modules` is a dict, so `sys.modules[k]` is a runtime lookup and + // `k` is a value. asking only whether the *base* (`sys`) is a module + // said yes here as readily as it does for `typing.List`, and the key + // was then folded to the type inferred for it — `sys.modules[str]`, + // which raises at runtime + check( + indoc! {" + import sys + + + class Point: + x: int + + + def main(): + point = Point() + print(sys.modules[type(point).__module__].__name__) + "}, + indoc! {" + import sys + + + class Point: + x: int + + + def main(): + point = Point() + print(sys.modules[type(point).__module__].__name__) + if __name__ == \"__main__\": + main() + "}, + ); + } + + #[test] + fn a_module_attribute_that_is_a_type_still_takes_type_arguments() { + // the other side of the same question: `typing.List` *is* a type, so its + // slice stays a type position and the fold still happens there + check( + "import typing\nc: typing.List[1 + 1]\n", + indoc! {" + from typing import Literal + import typing + c: typing.List[Literal[2]] + "}, + ); + } + #[test] fn attribute_type_folds_to_the_bound_member() { check_py312( diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 5b3defefd8..101903c6a0 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -38,7 +38,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Operator, Parameters, Stmt, TypeParam, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; /// the kind of type position currently being visited. lets visitors /// distinguish (e.g.) a syntactic annotation from an interior subtree @@ -283,14 +283,7 @@ impl TypePosWalker<'_> { let Some(types) = self.types else { return false; }; - match value { - Expr::Name(n) => types.subscript_is_known_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && types.subscript_is_known_type_context(value) } } diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index f51d10325e..3e50a6ab16 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -41,7 +41,7 @@ use ruff_python_ast::{self as ast, Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; /// dunder assignments whose values static readers (linters, type checkers, /// dataclass machinery) require to stay literal displays @@ -121,14 +121,10 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { // in a type-context subscript (`dict[str, int]`, legacy // `Callable[[int], str]`) the slice is a type expression, and // a display there is type syntax, not a value - let type_context = match subscript.value.as_ref() { - Expr::Name(name) => self.types.subscript_is_type_context(name), - Expr::Attribute(attribute) => match attribute.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - }; + let type_context = trailing_name(subscript.value.as_ref()).is_some() + && self + .types + .subscript_is_type_context(subscript.value.as_ref()); if type_context { self.visit_expr(&subscript.value); } else { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 8977cf6028..97ecbf5e16 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -71,19 +71,23 @@ pub(crate) enum CaptureKind { } pub(crate) trait TypeInfo { - /// whether `X[…]` where `X` is `name` treats the slice as type arguments. - /// returns `true` for unresolved / unknown names (covers builtins like - /// `list`, unimported sugar like `Union`) - fn subscript_is_type_context(&self, name: &ExprName) -> bool; - - /// stricter variant: only `true` when ty *resolved* `name` to a class / - /// generic / special form. unresolved names return `false`. used by + /// whether `X[…]` treats the slice as type arguments, where `X` is whatever + /// is being subscripted — a bare name (`list[int]`) or a dotted one + /// (`typing.List[int]`). returns `true` for unresolved / unknown values + /// (covers builtins like `list`, unimported sugar like `Union`) + /// + /// the question is asked of `X` itself rather than of its base. asking the + /// base "is this a module?" answers a different and much weaker question: + /// `typing.List` and `sys.modules` are both attributes of a module, but only + /// the first is a type — and reading `sys.modules[key]` as a type + /// application rewrote the key to the type it inferred for it + fn subscript_is_type_context(&self, value: &Expr) -> bool; + + /// stricter variant: only `true` when ty *resolved* the value to a class / + /// generic / special form. unresolved values return `false`. used by /// transforms that may fire on value-position subscripts (where an /// unresolved name should be treated as a runtime subscript, not a type) - fn subscript_is_known_type_context(&self, name: &ExprName) -> bool; - - /// whether `base.attr[…]` (base = a module or class) treats slice as type args - fn attr_base_is_type_context(&self, base: &ExprName) -> bool; + fn subscript_is_known_type_context(&self, value: &Expr) -> bool; fn is_function(&self, name: &ExprName) -> bool; @@ -543,8 +547,8 @@ impl TypeInfo for SemanticModel<'_> { .is_some_and(|ty| ty.is_attribute_type(self.db())) } - fn subscript_is_type_context(&self, name: &ExprName) -> bool { - match name.inferred_type(self) { + fn subscript_is_type_context(&self, value: &Expr) -> bool { + match value.inferred_type(self) { Some(ty) => ty.is_subscript_type_context(), // unresolved → assume type context (covers builtins like `list`, // unknown imports, basedpython sugar contexts) @@ -552,20 +556,13 @@ impl TypeInfo for SemanticModel<'_> { } } - fn subscript_is_known_type_context(&self, name: &ExprName) -> bool { - match name.inferred_type(self) { + fn subscript_is_known_type_context(&self, value: &Expr) -> bool { + match value.inferred_type(self) { Some(ty) => ty.is_subscript_type_context() && !ty.is_dynamic(), None => false, } } - fn attr_base_is_type_context(&self, base: &ExprName) -> bool { - match base.inferred_type(self) { - Some(ty) => ty.is_module_or_type(), - None => true, - } - } - fn is_function(&self, name: &ExprName) -> bool { name.inferred_type(self) .is_some_and(|ty| ty.as_function_literal().is_some()) @@ -1178,6 +1175,23 @@ impl TypeInfo for SemanticModel<'_> { /// diagnostic, where a symbolic arithmetic operation is shown as the expression it stands /// for (`I + 1`); emitting that would evaluate `_I + 1` on a `TypeVar` object at import, so /// the transpiler asks for the type it reduces to instead +/// the last component of a name or a dotted name — `List` for both `List` and +/// `typing.List` +/// +/// every spelling that takes type arguments is written one of those two ways, so +/// a transform looking for one asks for the trailing name and then asks +/// [`TypeInfo::subscript_is_type_context`] whether the whole thing is a type. the +/// two questions are separate on purpose: the name alone cannot tell +/// `typing.List` from an unrelated `mine.List`, and the type alone cannot tell +/// `Callable` from any other generic +pub(crate) fn trailing_name(expr: &Expr) -> Option<&str> { + match expr { + Expr::Name(name) => Some(name.id.as_str()), + Expr::Attribute(attribute) => Some(attribute.attr.id.as_str()), + _ => None, + } +} + fn display_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, diff --git a/crates/ruff_db/src/diagnostic/mod.rs b/crates/ruff_db/src/diagnostic/mod.rs index d438289e28..c33736d906 100644 --- a/crates/ruff_db/src/diagnostic/mod.rs +++ b/crates/ruff_db/src/diagnostic/mod.rs @@ -1051,6 +1051,31 @@ pub enum DiagnosticId { /// or remove the `include` option. EmptyInclude, + /// A negated `exclude` pattern that can never take effect. + /// + /// ## Why is this bad? + /// A negated pattern can only re-include a path whose parent directories are all still + /// walked. Once a directory is excluded nothing inside it is looked at, so a negation + /// pointing into that directory silently matches nothing. `dist` is excluded by default, + /// so `!dist/generated.py` never re-includes anything: the walk stops at `dist`. + /// + /// ## Example + /// ```toml + /// [src] + /// exclude = ["!dist/generated.py"] + /// ``` + /// + /// Use instead: + /// + /// ```toml + /// [src] + /// exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"] + /// ``` + /// + /// which re-includes the directory, excludes its contents again, then re-includes the + /// one file. + UnreachableExcludeNegation, + /// An override configuration is unnecessary because it applies to all files. /// /// ## Why is this bad? @@ -1159,6 +1184,7 @@ impl DiagnosticId { DiagnosticId::InvalidGlob => "invalid-glob", DiagnosticId::InvalidClassName => "invalid-class-name", DiagnosticId::EmptyInclude => "empty-include", + DiagnosticId::UnreachableExcludeNegation => "unreachable-exclude-negation", DiagnosticId::UnnecessaryOverridesSection => "unnecessary-overrides-section", DiagnosticId::UselessOverridesSection => "useless-overrides-section", DiagnosticId::DeprecatedSetting => "deprecated-setting", diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index bed525a5f2..31bcbdeb43 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -2073,7 +2073,13 @@ By default, ty excludes commonly ignored directories: - `**/venv/` You can override any default exclude by using a negated pattern. For example, -to re-include `dist` use `exclude = ["!dist"]` +to re-include `dist` use `exclude = ["!dist"]`, or `exclude = ["!**/dist/"]` to +re-include every `dist` directory rather than only the one at the project root. + +A negated pattern can only re-include something that is still walked, so it cannot +reach into a directory that is itself excluded. `exclude = ["!dist/generated.py"]` +re-includes nothing, because the walk stops at `dist`. Re-include the directory +first: `exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"]` **Default value**: `null` diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 328478b5c5..62ed8e698e 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::ffi::OsStr; use std::fs; use std::io::{self, Read}; @@ -222,21 +223,30 @@ pub(crate) fn cmd_run( for entry in &traceback_entries { let source = fs::read_to_string(&entry.by_path) .with_context(|| format!("could not read {}", entry.by_path.display()))?; - let name = entry + // the generated tree *is* the module tree, so the dotted name is the + // path within it — and it has to be dotted, because a class's + // `__module__` is read off the front of its type's `tp_name`. a file + // the tree gives no name to is left interpreted rather than compiled + // under a guessed one + let relative = entry .py_path - .file_stem() - .and_then(|stem| stem.to_str()) - .context("a source file has no usable module name")?; - if name == module { + .strip_prefix(tmp.path()) + .unwrap_or(&entry.py_path); + let Some(name) = dotted_module_name(relative) else { + continue; + }; + if name.dotted() == module { continue; } - let dir = entry.py_path.parent().unwrap_or(tmp.path()); let mut lowered = by_irbuild::module_from_source(&source, name, options.language); lowered.lines = Some(by_ir::function::LineTable::new( entry.by_path.display().to_string(), &source, )); - by_build::build_lowered(lowered, &source, &toolchain, dir, &options) + // the root of the generated tree, not the directory the `.py` landed + // in: the build lays the extension out at its module's own place, and + // handing it the leaf directory would nest the tree inside itself + by_build::build_lowered(lowered, &source, &toolchain, tmp.path(), &options) .with_context(|| format!("could not compile {}", entry.by_path.display()))?; built += 1; } @@ -310,6 +320,89 @@ fn module_relative_path(roots: &[PathBuf], root: &Path, bpy: &Path) -> PathBuf { .with_extension("py") } +/// The dotted module name a file laid out at `relative` will be imported under. +/// +/// The tree the generated python is written into *is* the module tree — every +/// file lands at [`module_relative_path`] — so the name is that path with its +/// separators turned into dots. `pkg/__init__.py` is the package `pkg` itself, +/// which is the name a class defined in it reports as its `__module__`. +/// +/// `None` when the path names no module: nothing but `__init__.py` at the root, +/// or a component that is not plain text. A caller with no name has nothing to +/// compile, and guessing one would be worse than saying so. +fn dotted_module_name(relative: &Path) -> Option { + let mut components: Vec<&str> = Vec::new(); + for component in relative.components() { + let std::path::Component::Normal(name) = component else { + return None; + }; + components.push(name.to_str()?); + } + let last = components.pop()?; + let stem = last.strip_suffix(".py").unwrap_or(last); + // a package's `__init__` is not a module beside the package, it *is* the + // package — so a class written in it belongs to the package's own name + let is_package = stem == "__init__"; + if !is_package { + components.push(stem); + } + if components.is_empty() { + return None; + } + let dotted = components.join("."); + Some(if is_package { + by_ir::ModuleName::package(dotted) + } else { + by_ir::ModuleName::new(dotted) + }) +} + +/// The dotted module name `file` is imported under, as the project resolves it. +/// +/// The resolver is what knows this, and nothing simpler will do: it accounts for +/// the search paths (`src/pkg/m.py` in a src-layout project is `pkg.m`, not +/// `src.pkg.m`), for a `pkg/__init__.py` whose module is `pkg`, and for a +/// namespace package, which a walk looking for `__init__.py` would stop short of. +/// It also refuses a directory that merely holds `.py` files — cpython's own +/// `config-3.13-darwin` is not a package, and its name is not even an identifier. +/// +/// `None` when no search path reaches the file. Such a file has no dotted name to +/// be had: the only way to import it is from its own directory, under its stem. +fn resolved_module_name(db: &ProjectDatabase, file: ruff_db::files::File) -> Option { + let program_file = ty_python_semantic::Db::program_file(db, file); + let module = ty_module_resolver::file_to_module(db, program_file.resolver_file(db))?; + Some(module.name(db).to_string()) +} + +/// The name `path` is compiled under — dotted, and knowing whether it is a +/// package's. +/// +/// The *dotted* name because it is what the emitted types carry: cpython reads a +/// class's `__module__` off the front of its `tp_name`, so a class in +/// `tkinter/m.py` compiled as plain `m` reports a module nothing can look up — +/// and `dataclasses` does exactly that lookup. +/// +/// Package or not because the two are written to different files. The resolver +/// names a package after its directory, so an `__init__.py` it resolved is that +/// package, and its artefact is the `__init__` inside the directory. A file the +/// resolver could not reach has no dotted name at all: it falls back to its stem, +/// which is the only name it could be imported under, from its own directory. +fn compiled_module_name( + db: &ProjectDatabase, + path: &Path, + file: ruff_db::files::File, +) -> anyhow::Result { + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .context("a source file has no usable module name")?; + Ok(match resolved_module_name(db, file) { + Some(resolved) if stem == "__init__" => by_ir::ModuleName::package(resolved), + Some(resolved) => by_ir::ModuleName::new(resolved), + None => by_ir::ModuleName::new(stem), + }) +} + /// The project's `run.main` entry point, if one is configured. fn configured_main(db: &ProjectDatabase) -> Option { let options = db.project().metadata(db).options(); @@ -435,7 +528,40 @@ pub(crate) fn cmd_compile( // gradual, and `--no-any` would then fail on noise // `compile` embeds fallback source produced by the untyped transpile, which // takes no db, so the rebuilder the other commands thread through is unused here - let (db, handles, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + + // the database holds the whole project so a type imported from a sibling + // resolves, but only the files that were *asked for* are checked and emitted. + // compiling the project regardless of the arguments is not a harmless + // superset: it costs every other module's build time, it fails the command + // for a diagnostic in a file nobody named, and — because the argument is + // ignored rather than rejected — it silently compiles a file beside the one + // under test, which has already invalidated one delta-debugging run here + let requested: Vec = sources + .iter() + .map(|source| { + if source.is_absolute() { + source.clone() + } else { + cwd.join(source) + } + }) + .map(|source| source.canonicalize().unwrap_or(source)) + .collect(); + let handles: Vec<_> = project + .iter() + .filter(|(path, _)| { + files.is_empty() + || requested.iter().any(|wanted| { + *wanted == **path || path.canonicalize().is_ok_and(|p| p == *wanted) + }) + }) + .cloned() + .collect(); + if handles.is_empty() { + eprintln!("no .by or .py files found"); + return Ok(ExitStatus::Success); + } // a source with nothing to lower blocks, the way it does for `build` and // `transpile` — it could not be parsed, or could not be read at all. a *type* @@ -459,13 +585,32 @@ pub(crate) fn cmd_compile( render_diagnostics(&db, &diagnostics)?; } + // what each source will be compiled as, worked out before anything is written: + // two sources that land on the same artefact used to leave only the second, and + // nothing said so + let mut names: Vec = Vec::with_capacity(handles.len()); + let mut claimed: HashMap = HashMap::new(); for (path, file) in &handles { + let name = compiled_module_name(&db, path, *file)?; + // keyed on the artefact rather than on the name, because the artefact is + // what would be overwritten — and a file the resolver cannot name falls + // back to its stem, which two directories can share + let artifact = name.relative_path(""); + if let Some(first) = claimed.insert(artifact, path.as_path()) { + anyhow::bail!( + "`{}` and `{}` would both be compiled as the module `{}`, \ + and the second would replace the first's artifact", + first.display(), + path.display(), + name.dotted() + ); + } + names.push(name); + } + + for ((path, file), name) in handles.iter().zip(names) { let source = fs::read_to_string(path) .with_context(|| format!("could not read {}", path.display()))?; - let module = path - .file_stem() - .and_then(|stem| stem.to_str()) - .context("a source file has no usable module name")?; let program_file = ty_python_semantic::Db::program_file(&db, *file); let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); @@ -480,7 +625,7 @@ pub(crate) fn cmd_compile( &model.program_environment(), &model, parsed.suite(), - module, + name, options.language.unique_loop_bindings(), ); // the real path, so a `#line` in the generated C resolves for a debugger @@ -1045,6 +1190,29 @@ const BY_SOURCES: &[&str] = &["by", "byi"]; /// output directory rather than beside the source, so a `.py` is an input const COMPILABLE_SOURCES: &[&str] = &["by", "byi", "py"]; +/// The [`NON_SOURCE_DIRS`] entries that ty's own `src.exclude` defaults don't already drop. +/// +/// [`is_hidden_within`] runs over files that have *already* passed the project's file filter, +/// so for a name ty excludes by default — `venv`, `dist`, `node_modules`, `.tox`, … — a file +/// can only have reached it because the configuration deliberately re-included the directory +/// with a negated pattern, which `src.exclude` documents as the way to override a default. +/// Re-dropping such a file here would quietly undo that, and it's why a project could not +/// compile a module of its own that happens to live in a directory named `venv`. +/// +/// What's left are the names ty has no default opinion about, where this walk is the only +/// thing keeping a dependency tree or a build output out of the emitted set. The unfiltered +/// [`NON_SOURCE_DIRS`] still applies to [`may_contain_sources`], which walks the file system +/// directly and never sees the project configuration at all. +const NON_SOURCE_DIRS_TY_ALLOWS: &[&str] = &[ + "env", + ".env", + "site-packages", + "__pycache__", + ".pytest_cache", + "build", + "out", +]; + /// Whether `path` sits inside a hidden or build-output directory under `root`. fn is_hidden_within(path: &Path, root: &Path) -> bool { let Ok(relative) = path.strip_prefix(root) else { @@ -1055,7 +1223,7 @@ fn is_hidden_within(path: &Path, root: &Path) -> bool { .into_iter() .flat_map(Path::components) .filter_map(|component| component.as_os_str().to_str()) - .any(|name| name.starts_with('.') || NON_SOURCE_DIRS.contains(&name)) + .any(|name| name.starts_with('.') || NON_SOURCE_DIRS_TY_ALLOWS.contains(&name)) } /// Build a project db rooted at `cwd`, returning it alongside the @@ -1287,7 +1455,10 @@ pub(crate) fn cmd_version_by(output_format: crate::args::HelpFormat) -> ExitStat #[cfg(test)] mod tests { - use super::{is_hidden_within, module_relative_path, reverse_dir, reverse_dir_converting}; + use super::{ + dotted_module_name, is_hidden_within, module_relative_path, reverse_dir, + reverse_dir_converting, + }; use crate::ExitStatus; use by_transforms::config::Config; use std::path::{Path, PathBuf}; @@ -1313,6 +1484,37 @@ mod tests { ); } + /// a compiled module carries its name into every type it emits, and cpython + /// reads a class's `__module__` off the front of that — so the name of a file + /// inside the tree is the whole path to it, not the file's own stem + #[test] + fn a_file_inside_the_tree_is_named_for_its_whole_path() { + assert_eq!( + dotted_module_name(Path::new("pkg/sub/main.py")), + Some(by_ir::ModuleName::new("pkg.sub.main")) + ); + assert_eq!( + dotted_module_name(Path::new("top.py")), + Some(by_ir::ModuleName::new("top")) + ); + } + + /// a package's `__init__` is not a module beside the package, it *is* the + /// package — which is the module a class written in it belongs to + #[test] + fn a_packages_init_is_named_for_the_package() { + let name = dotted_module_name(Path::new("pkg/__init__.py")); + assert_eq!(name, Some(by_ir::ModuleName::package("pkg"))); + // and its artefact is the `__init__` inside the package, not a file + // called `pkg` beside it — that is the only place cpython's finder looks + assert_eq!( + name.map(|name| name.relative_path(".so")), + Some(PathBuf::from("pkg/__init__.so")) + ); + // and at the root there is no package for it to be, so there is no name + assert_eq!(dotted_module_name(Path::new("__init__.py")), None); + } + /// a root that shares no prefix with the file — which is what `canonicalize` /// and `current_dir` disagreeing produced on windows — must not leave the /// path absolute: joining that onto the output directory discards the output @@ -1340,6 +1542,10 @@ mod tests { assert!(!is_hidden_within(Path::new("/p/src/pkg/main.by"), root)); // the file's own name is not a directory component assert!(!is_hidden_within(Path::new("/p/.hidden.by"), root)); + // a name ty excludes by default is left to the project filter, so that a + // negated `src.exclude` pattern re-including it isn't quietly undone here + assert!(!is_hidden_within(Path::new("/p/venv/__init__.by"), root)); + assert!(!is_hidden_within(Path::new("/p/dist/main.by"), root)); } /// a source that declares its own encoding converts like any other, and the diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 3ba786aa0d..8c9260df37 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -48,6 +48,179 @@ fn run_transpile(source: &str, extra_args: &[&str]) -> String { String::from_utf8(output.stdout).unwrap() } +#[test] +fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { + // `by compile a.py` used to compile every source in the project and ignore the + // argument entirely. that is not a harmless superset: it costs every other + // module's build time, it fails the command for a diagnostic in a file nobody + // named, and it silently compiles a file sitting beside the one under test — + // which invalidated a delta-debugging run whose original was in the same + // directory as each candidate + // + // the database still holds the whole project, because a type imported from a + // sibling has to resolve. that is what `lib.py` is here to prove: it is never + // compiled, and `wanted.py` still lowers `Point` rather than declining + let dir = std::env::temp_dir().join("by_cli_only_named"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + std::fs::write( + dir.join("lib.py"), + "class Point:\n def __init__(self) -> None:\n self.x: int = 7\n", + ) + .unwrap(); + std::fs::write( + dir.join("wanted.py"), + "from lib import Point\n\n\ndef go() -> int:\n p = Point()\n return p.x\n", + ) + .unwrap(); + std::fs::write( + dir.join("other.py"), + "def unrelated() -> int:\n return 2\n", + ) + .unwrap(); + + let out = dir.join("out"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "wanted.py", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + + assert!(out.join("wanted.c").exists(), "the named file is compiled"); + assert!( + !out.join("lib.c").exists() && !out.join("other.c").exists(), + "a file that was not named is not compiled" + ); + + // the cross-module type resolved: a declined body would not carry the + // attribute read at all + let emitted = std::fs::read_to_string(out.join("wanted.c")).expect("the C is readable"); + assert!( + emitted.contains("by_wanted_go"), + "`go` lowered natively, so `Point` resolved out of the uncompiled sibling" + ); +} + +/// write a project of package members under `dir`, each answering with its own +/// dotted name +fn write_package_project(dir: &std::path::Path) { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir.join("pkg/sub")).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + for (path, tag) in [ + ("pkg/__init__.py", 1), + ("pkg/sub/__init__.py", 2), + ("pkg/dup.py", 3), + ("pkg/sub/dup.py", 4), + ] { + std::fs::write( + dir.join(path), + format!("def tag() -> int:\n return {tag}\n"), + ) + .unwrap(); + } +} + +#[test] +fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { + // `by compile -o out` used to write every artefact flat, named after the + // module's last component. two members of a package sharing a last component + // then wrote the same file and the second silently won — and *no* package + // member's artefact was importable under the name it had been compiled as, + // because a flat `dup.so` can only ever be imported as `dup` + let dir = std::env::temp_dir().join("by_cli_package_tree"); + write_package_project(&dir); + + let out = dir.join("o"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + + // four sources, four artefacts — a package's own file is the `__init__` inside + // its directory, which is the only name cpython's finder looks for + for relative in [ + "pkg/__init__.c", + "pkg/sub/__init__.c", + "pkg/dup.c", + "pkg/sub/dup.c", + ] { + assert!(out.join(relative).exists(), "{relative} was written"); + } + // and nothing named after a last component alone + assert!( + !out.join("dup.c").exists() && !out.join("sub.c").exists() && !out.join("pkg.c").exists() + ); + + // the two `dup` members are distinct modules, not one file written twice + let first = fs::read_to_string(out.join("pkg/dup.c")).unwrap(); + let second = fs::read_to_string(out.join("pkg/sub/dup.c")).unwrap(); + assert!(first.contains("by_pkg_dup_tag"), "{first}"); + assert!(second.contains("by_pkg_sub_dup_tag"), "{second}"); +} + +#[test] +fn compile_refuses_two_sources_that_would_write_the_same_artifact() { + // laying the output out as the module tree settles the collision between two + // package members, but not this one: neither directory here has a name python + // could import, so neither file has a dotted name and both fall back to their + // stem. one artefact would be written twice and only the second kept, which is + // the silent loss the tree was meant to end — so it is refused before anything + // is written rather than half-performed + let dir = std::env::temp_dir().join("by_cli_artifact_clash"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("a-one")).unwrap(); + std::fs::create_dir_all(dir.join("b-two")).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + std::fs::write(dir.join("a-one/m.py"), "def tag() -> int:\n return 1\n").unwrap(); + std::fs::write(dir.join("b-two/m.py"), "def tag() -> int:\n return 2\n").unwrap(); + + let out = dir.join("o"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!(!result.status.success(), "the clash is refused"); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!( + stderr.contains("would both be compiled as the module `m`"), + "{stderr}" + ); + // and it said so before writing either one + assert!(!out.join("m.c").exists(), "{stderr}"); +} + #[test] fn compile_transpiles_the_fallback_with_the_lowering_options_it_was_given() { // a declined function *runs* from the embedded source, so `by compile` has to diff --git a/crates/ty/tests/cli/file_selection.rs b/crates/ty/tests/cli/file_selection.rs index 841e259434..a38fe1f61c 100644 --- a/crates/ty/tests/cli/file_selection.rs +++ b/crates/ty/tests/cli/file_selection.rs @@ -544,6 +544,105 @@ fn remove_default_exclude() -> anyhow::Result<()> { Ok(()) } +/// A negation that names the directory re-includes it without needing the `**/` the default +/// exclude is written with, which is the spelling the `exclude` documentation gives. +#[test] +fn remove_default_exclude_unqualified() -> anyhow::Result<()> { + let case = CliTest::with_files([( + "dist/generated.py", + r#" + print(another_undefined_var) # error: unresolved-reference + "#, + )])?; + + for pattern in ["!dist", "!dist/", "!./dist"] { + case.write_file("ty.toml", &format!("[src]\nexclude = [\"{pattern}\"]\n"))?; + + insta::allow_duplicates! { + assert_cmd_snapshot!(case.command(), @" + success: false + exit_code: 1 + ----- stdout ----- + error[unresolved-reference]: Name `another_undefined_var` used when not defined + --> dist/generated.py:2:7 + | + 2 | print(another_undefined_var) # error: unresolved-reference + | ^^^^^^^^^^^^^^^^^^^^^ + + Found 1 diagnostic + + ----- stderr ----- + "); + } + } + + Ok(()) +} + +/// A negation can only re-include something the walk still reaches. Pointing one into a +/// directory that is itself excluded matches nothing at all, so it's reported rather than +/// silently doing nothing. +#[test] +fn negation_into_an_excluded_directory_warns() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("src/main.py", "print('ok')"), + ("dist/generated.py", "print(dist_var)"), + ("dist/keep.py", "print(keep_var)"), + ])?; + + case.write_file( + "ty.toml", + r#" + [src] + exclude = ["!dist/keep.py"] + "#, + )?; + + assert_cmd_snapshot!(case.command(), @r#" + success: false + exit_code: 1 + ----- stdout ----- + warning[unreachable-exclude-negation]: Negated pattern `!dist/keep.py` has no effect + --> ty.toml:3:12 + | + 3 | exclude = ["!dist/keep.py"] + | ^^^^^^^^^^^^^^^ This pattern can never match + info: `dist` is excluded, so nothing inside it is ever reached + info: Re-include the directory first by adding `!**/dist/` + + Found 1 diagnostic + + ----- stderr ----- + "#); + + // Re-including the directory first makes the same negation reachable, so the warning goes + // away and only `keep.py` comes back. + case.write_file( + "ty.toml", + r#" + [src] + exclude = ["!**/dist/", "**/dist/**", "!dist/keep.py"] + "#, + )?; + + assert_cmd_snapshot!(case.command(), @" + success: false + exit_code: 1 + ----- stdout ----- + error[unresolved-reference]: Name `keep_var` used when not defined + --> dist/keep.py:1:7 + | + 1 | print(keep_var) + | ^^^^^^^^ + + Found 1 diagnostic + + ----- stderr ----- + "); + + Ok(()) +} + /// Test that configuration excludes can be removed via CLI negation #[test] fn cli_removes_config_exclude() -> anyhow::Result<()> { diff --git a/crates/ty_project/src/glob/exclude.rs b/crates/ty_project/src/glob/exclude.rs index c7f6e625b9..8d8a63ce6c 100644 --- a/crates/ty_project/src/glob/exclude.rs +++ b/crates/ty_project/src/glob/exclude.rs @@ -291,3 +291,88 @@ impl GitignoreBuilder { Ok(self) } } + +#[cfg(test)] +mod tests { + use ruff_db::system::SystemPath; + + use crate::GlobFilterCheckMode; + use crate::glob::exclude::{ExcludeFilter, ExcludeFilterBuilder}; + use crate::glob::{PortableGlobKind, PortableGlobPattern}; + use crate::metadata::options::DEFAULT_SRC_EXCLUDES; + + /// Builds an exclude filter the same way `build_exclude_filter` does: the defaults first, + /// anchored at the file system root, then the user's patterns anchored at the project root. + fn filter_with_defaults( + user_patterns: impl IntoIterator, + ) -> ExcludeFilter { + let mut builder = ExcludeFilterBuilder::new(); + + for pattern in DEFAULT_SRC_EXCLUDES { + builder + .add( + &PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute(""), + ) + .unwrap(); + } + + for pattern in user_patterns { + builder + .add( + &PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute("/project"), + ) + .unwrap(); + } + + builder.build().unwrap() + } + + #[track_caller] + fn assert_excluded(patterns: impl IntoIterator, path: &str) { + let filter = filter_with_defaults(patterns); + assert!( + filter.match_directory(SystemPath::new(path), GlobFilterCheckMode::TopDown), + "`{path}` should be excluded" + ); + } + + #[track_caller] + fn assert_not_excluded(patterns: impl IntoIterator, path: &str) { + let filter = filter_with_defaults(patterns); + assert!( + !filter.match_directory(SystemPath::new(path), GlobFilterCheckMode::TopDown), + "`{path}` should not be excluded" + ); + } + + /// A negation that names the directory itself re-includes it, whichever way it's spelled. + #[test] + fn negation_re_includes_a_default_excluded_directory() { + assert_excluded([], "/project/dist"); + + assert_not_excluded(["!dist"], "/project/dist"); + assert_not_excluded(["!dist/"], "/project/dist"); + assert_not_excluded(["!**/dist/"], "/project/dist"); + assert_not_excluded(["!./dist"], "/project/dist"); + assert_not_excluded(["!/project/dist"], "/project/dist"); + } + + /// Patterns are anchored at the project root, so an unqualified negation doesn't reach a + /// nested directory of the same name. `**/` does. + #[test] + fn negation_of_a_nested_directory_needs_a_wildcard() { + assert_excluded(["!dist"], "/project/pkg/dist"); + assert_not_excluded(["!**/dist/"], "/project/pkg/dist"); + } + + /// `dist/**` names the contents of `dist`, not `dist` itself, so negating it leaves the + /// directory excluded and the contents out of reach. + #[test] + fn negating_the_contents_leaves_the_directory_excluded() { + assert_excluded(["!dist/**"], "/project/dist"); + } +} diff --git a/crates/ty_project/src/glob/portable.rs b/crates/ty_project/src/glob/portable.rs index 3e2cd568cb..7e249411b3 100644 --- a/crates/ty_project/src/glob/portable.rs +++ b/crates/ty_project/src/glob/portable.rs @@ -147,8 +147,15 @@ impl<'a> PortableGlobPattern<'a> { } if pattern.starts_with('/') { + // An already-rooted pattern needs no anchoring, but it does need its `!` back: + // without it `!/root/src` reads as an exclude of `/root/src`, the opposite of + // what was written. return AbsolutePortableGlobPattern { - absolute: pattern.to_string(), + absolute: if negated { + format!("!{pattern}") + } else { + pattern.to_string() + }, relative: self.pattern.to_string(), }; } @@ -231,6 +238,64 @@ impl AbsolutePortableGlobPattern { pub(crate) fn relative(&self) -> &str { &self.relative } + + /// Returns `true` if this is a negated (`!`-prefixed) exclude pattern. + pub(crate) fn is_negated(&self) -> bool { + self.absolute.starts_with('!') + } + + /// The deepest directory that has to be walked for this pattern to match anything. + /// + /// A glob can only match paths that start with the components before its first wildcard, + /// so those components bound how deep the pattern can reach. `/root/dist/**` can only ever + /// match something inside `/root/dist`, and `/root/dist/generated.py` can only match a file + /// in that same directory, so both need `/root/dist` to be walked. A pattern that names a + /// path outright needs only that path's parent, which is why `/root/dist` itself returns + /// `/root`: re-including `dist` doesn't require walking into it. + /// + /// Returns `None` for a pattern that isn't anchored to any directory at all, such as + /// `**/dist/` before it's made absolute. + pub(crate) fn required_directory(&self) -> Option<&SystemPath> { + let pattern = self.absolute.strip_prefix('!').unwrap_or(&self.absolute); + + let wildcard_start = pattern + .split('/') + .scan(0usize, |offset, component| { + let start = *offset; + *offset += component.len() + 1; + Some((start, component)) + }) + .find(|(_, component)| is_wildcard_component(component)) + .map(|(start, _)| start); + + match wildcard_start { + // Everything before the first wildcard is literal, so that prefix is the deepest + // directory the pattern can be anchored to. The trailing `/` is dropped with it. + Some(start) => { + let prefix = &pattern[..start.saturating_sub(1)]; + (!prefix.is_empty()).then(|| SystemPath::new(prefix)) + } + // No wildcard at all: the pattern names one path, so only its parent is walked. + None => SystemPath::new(pattern).parent(), + } + } +} + +/// Returns `true` if `component` contains an unescaped glob metacharacter. +fn is_wildcard_component(component: &str) -> bool { + let mut chars = component.chars(); + + while let Some(c) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '*' | '?' | '[' => return true, + _ => {} + } + } + + false } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -393,6 +458,45 @@ mod tests { assert_absolute_path("./src", "/root", "/root/src"); } + /// A pattern that's already rooted skips anchoring, but it still has to keep its `!`: + /// dropping it turns a re-include into an exclude of the same path. + #[test] + fn negated_absolute_pattern_keeps_its_negation() { + assert_absolute_path("!/src", "/root", "!/src"); + assert_absolute_path("!./src", "/root", "!/root/src"); + } + + #[track_caller] + fn assert_required_directory( + pattern: &str, + relative_to: impl AsRef, + expected: &str, + ) { + let pattern = PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute(relative_to); + assert_eq!( + pattern.required_directory().map(SystemPath::as_str), + (!expected.is_empty()).then_some(expected) + ); + } + + #[test] + fn required_directory() { + // A pattern that names one path only needs that path's parent walked. + assert_required_directory("!dist", "/root", "/root"); + assert_required_directory("!dist/", "/root", "/root"); + assert_required_directory("!dist/generated.py", "/root", "/root/dist"); + // Everything before the first wildcard bounds how deep the pattern can reach. + assert_required_directory("!dist/**", "/root", "/root/dist"); + assert_required_directory("!dist/*.py", "/root", "/root/dist"); + assert_required_directory("!**/dist/", "/root", "/root"); + // An escaped `*` is a literal, not a wildcard. + assert_required_directory(r"!dist/\*/x.py", "/root", r"/root/dist/\*"); + // Nothing to anchor to. + assert_required_directory("**/dist/", "", ""); + } + #[test] #[cfg(windows)] fn absolute_pattern_windows() { diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index 15aca23cd7..39c2d5cf60 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -1,5 +1,9 @@ use crate::Db; -use crate::glob::{ExcludeFilter, IncludeExcludeFilter, IncludeFilter, PortableGlobKind}; +use crate::GlobFilterCheckMode; +use crate::glob::{ + AbsolutePortableGlobPattern, ExcludeFilter, IncludeExcludeFilter, IncludeFilter, + PortableGlobKind, +}; use crate::metadata::python_version::SupportedPythonVersion; use crate::metadata::settings::{OverrideSettings, SrcSettings}; @@ -1015,7 +1019,13 @@ pub struct SrcOptions { /// - `**/venv/` /// /// You can override any default exclude by using a negated pattern. For example, - /// to re-include `dist` use `exclude = ["!dist"]` + /// to re-include `dist` use `exclude = ["!dist"]`, or `exclude = ["!**/dist/"]` to + /// re-include every `dist` directory rather than only the one at the project root. + /// + /// A negated pattern can only re-include something that is still walked, so it cannot + /// reach into a directory that is itself excluded. `exclude = ["!dist/generated.py"]` + /// re-includes nothing, because the walk stops at `dist`. Re-include the directory + /// first: `exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"]` #[option( default = r#"null"#, value_type = r#"list[str]"#, @@ -1052,6 +1062,7 @@ impl SrcOptions { self.exclude.as_ref(), DEFAULT_SRC_EXCLUDES, GlobFilterContext::SrcRoot, + diagnostics, )?; let files = IncludeExcludeFilter::new(include, exclude); @@ -1276,6 +1287,7 @@ fn build_exclude_filter( exclude_patterns: Option<&RangedValue>>, default_patterns: &[&str], context: GlobFilterContext, + diagnostics: &mut Vec, ) -> Result> { use crate::glob::{ExcludeFilterBuilder, PortableGlobPattern}; @@ -1290,12 +1302,19 @@ fn build_exclude_filter( }); } + // Held on to so that, once the filter is built, every negation can be checked against the + // whole pattern set — including the negations that come after it. + let mut negations = Vec::new(); + // Add user-specified excludes if let Some(exclude_patterns) = exclude_patterns { for exclude in exclude_patterns { - exclude + let pattern = exclude .absolute(project_root, system, PortableGlobKind::Exclude) - .and_then(|pattern| Ok(excludes.add(&pattern)?)) + .and_then(|pattern| { + excludes.add(&pattern)?; + Ok(pattern) + }) .map_err(|err| { let diagnostic = OptionDiagnostic::new( DiagnosticId::InvalidGlob, @@ -1311,10 +1330,14 @@ fn build_exclude_filter( err, ) })?; + + if pattern.is_negated() { + negations.push((exclude, pattern)); + } } } - excludes.build().map_err(|_| { + let filter = excludes.build().map_err(|_| { let diagnostic = OptionDiagnostic::new( DiagnosticId::InvalidGlob, format!( @@ -1328,7 +1351,63 @@ fn build_exclude_filter( "Please open an issue on the ty repository \ and share the patterns that caused the error.", ))) - }) + })?; + + for (exclude, pattern) in negations { + let Some(blocked_by) = unreachable_negation_cause(&filter, &pattern) else { + continue; + }; + + // Paths in a configuration file read better relative to the project they configure. + let blocked_by_display = blocked_by.strip_prefix(project_root).unwrap_or(blocked_by); + + let mut diagnostic = OptionDiagnostic::new( + DiagnosticId::UnreachableExcludeNegation, + format!("Negated pattern `{exclude}` has no effect"), + Severity::Warning, + ) + .sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format!("`{blocked_by_display}` is excluded, so nothing inside it is ever reached"), + )); + + if let Some(name) = blocked_by.file_name() { + diagnostic = diagnostic.sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format!("Re-include the directory first by adding `!**/{name}/`"), + )); + } + + if let Some(source_file) = exclude.value().source().file() + && let Ok(file) = system_path_to_file(db, source_file) + { + diagnostic = diagnostic.with_annotation(Some( + Annotation::primary(Span::from(file).with_optional_range(exclude.value().range())) + .message("This pattern can never match"), + )); + } + + diagnostics.push(diagnostic); + } + + Ok(filter) +} + +/// Returns the excluded directory that stops `negation` from ever re-including anything. +/// +/// A negation only takes effect if the walk actually reaches the paths it matches, and the walk +/// stops at the first excluded directory. So the pattern is dead if any directory it has to be +/// reached through is excluded — the shallowest such directory is the one reported, because that +/// is where the walk really stops. +fn unreachable_negation_cause<'a>( + filter: &ExcludeFilter, + negation: &'a AbsolutePortableGlobPattern, +) -> Option<&'a SystemPath> { + negation + .required_directory()? + .ancestors() + .filter(|directory| filter.match_directory(directory, GlobFilterCheckMode::TopDown)) + .last() } /// Context for filter operations, used in error messages @@ -2600,6 +2679,7 @@ impl ToOverride for RangedValue { self.exclude.as_ref(), &[], GlobFilterContext::Overrides, + diagnostics, )?; let files = IncludeExcludeFilter::new(include, exclude); diff --git a/crates/ty_python_semantic/resources/mdtest/attributes.md b/crates/ty_python_semantic/resources/mdtest/attributes.md index 0185f3ad60..5e02dd795d 100644 --- a/crates/ty_python_semantic/resources/mdtest/attributes.md +++ b/crates/ty_python_semantic/resources/mdtest/attributes.md @@ -4090,7 +4090,11 @@ class ManyCycles2: self.x3 = [1] def f1(self: "ManyCycles2"): - # revealed: list[int] | list[Divergent] | UnsafeUnion[list[int], list[Divergent]] + # the union carries `UnsafeUnion[list[int], list[Divergent]]` twice over, and a + # three-element variant of the same thing. that redundancy is the recovery's, not the + # program's — a union of a cycle's rounds is not being collapsed to one element per + # distinct type. pinned as it stands so a change to it is visible rather than silent + # revealed: list[int] | list[Divergent] | UnsafeUnion[list[int], list[Divergent]] | UnsafeUnion[list[int], list[Divergent]] | UnsafeUnion[list[int], list[Divergent], list[Divergent]] | list[Divergent] reveal_type(self.x3) self.x1 = [self.x2] + [self.x3] diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md index 2c826d4547..0a12fe5103 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md @@ -567,6 +567,50 @@ def f(x): f("anything") # ok ``` +### a parameter its own body rebinds says nothing + +a name bound more than once cannot stand for one value, which is already why a reassigned local +contributes nothing. a parameter is no different once its own body rebinds it: below the rebinding +the name is whatever the rebinding produced, so what is done with it there requires nothing of what +the caller passed + +keeping the requirements collected *above* the rebinding would not do either. walking a traceback +requires only that the argument have a `tb_next`, so the rebinding lands on that member's type — +which is `object`, because nothing said what it holds — and the read below it then fails against the +bound the function itself produced + +```py +def deepest(tb): + if tb.tb_next: + tb = tb.tb_next + return tb.tb_frame + +deepest("anything") # ok +``` + +the same holds when the member is a method, which is the shape most of these take + +```py +def rebound(x): + x.foo() + x = x.foo() + x.foo() + +rebound("anything") # ok +``` + +the rebinding does not have to be reachable, or to come from the parameter, for the name to stop +standing for one value + +```py +def maybe(x, flag): + x.foo() + if flag: + x = 1 + +maybe("anything", True) # ok +``` + ### a recursive call does not constrain ```py @@ -1322,6 +1366,34 @@ def recur(a): reveal_type(recur([])) # revealed: list[Divergent] ``` +### recursion no marker survives + +that marker only survives while the type is *built*. a body that hands itself to a generic call gets +its return type back out of that call's solve, which leaves a concrete type behind with nothing to +fold on — `map(g, n)` is `map[map[...]]` receding by one constructor every round. the recursion is +recognised by a round adding nothing but depth, and collapsed onto the marker one level in: + +```py +def g(n): + return map(g, n) + +reveal_type(g([])) # revealed: map[Divergent] +``` + +the same holds when the recursion goes round two functions: + +```py +def to(n): + return map(fro, n) + +def fro(n): + return map(to, n) + +# the mutual case settles a level shallower than the single-function one above, which is the +# recursion being recognised on a different round rather than anything the program says +reveal_type(to([])) # revealed: map[map[Never]] +``` + ### recursion that grows a tuple a body that concatenates onto its own result adds an element every round, so no tuple length is the @@ -1351,6 +1423,26 @@ def inner(b): reveal_type(outer([])) # revealed: tuple[Literal[1], ...] ``` +### a tuple whose elements are the round that built them + +an attribute taken apart and put back together is defined in terms of itself, so its elements are +the marker. giving up the length would union those elements into one — and an element standing for +the cycle is precisely what the marker replaces, so the widened tuple has to be handed back through +the marker rather than around it. a round that widened and a round that marked would otherwise each +undo the other, and the two answers would alternate without either ever being reached: + +```py +class C: + def __init__(self): + self._t = (0, 0) + + def f(self): + a, b = self._t + reveal_type(a) # revealed: Divergent + self._t = (a, b) + reveal_type(self._t) # revealed: tuple[Divergent, Divergent] +``` + ### generators ```py diff --git a/crates/ty_python_semantic/resources/mdtest/cycle.md b/crates/ty_python_semantic/resources/mdtest/cycle.md index d3b92efb80..1927cc9026 100644 --- a/crates/ty_python_semantic/resources/mdtest/cycle.md +++ b/crates/ty_python_semantic/resources/mdtest/cycle.md @@ -230,6 +230,11 @@ class D: ### Lambdas +all four show the default one layer deeper than the parameter it is the default of. the two +positional ones used to stop a layer earlier, because the expected type carried into the lambda +folded them back onto the marker — a context holding the cycle's own marker no longer earns a query +key of its own, so they now read the same way the keyword-only two always have: + ```py class C: def f(self: "C"): @@ -238,13 +243,13 @@ class C: self.c = lambda positional_only=self.c, /: positional_only self.d = lambda *, kw_only=self.d: kw_only - # revealed: (positional: Divergent = ...) -> Divergent + # revealed: (positional: (positional: Divergent = ...) -> Divergent = ...) -> Divergent reveal_type(self.a) # revealed: (*, kw_only: (*, kw_only: Divergent = ...) -> Divergent = ...) -> Divergent reveal_type(self.b) - # revealed: (positional_only: Divergent = ..., /) -> Divergent + # revealed: (positional_only: (positional_only: Divergent = ..., /) -> Divergent = ..., /) -> Divergent reveal_type(self.c) # revealed: (*, kw_only: (*, kw_only: Divergent = ...) -> Divergent = ...) -> Divergent @@ -621,3 +626,281 @@ class Reader: reveal_type(self.decoder) # revealed: None | Unknown reveal_type(self.used) # revealed: int | cookie@seek ``` + +## an attribute rebuilt out of its own elements + +one method seeds an attribute with a fixed-length tuple and another rebuilds it out of an element it +reads back out, so the attribute is defined in terms of itself. both bindings are the same length, +so the widening that gives up a *growing* length has nothing to give up — but the rebuilt tuple +still has an element standing for the cycle, and an element standing for the cycle is exactly what +the divergence marker replaces. the widened form and the marked form each undo the other, so unless +the widening is handed back through the marker rather than around it the two alternate with period +two and neither is ever reached. + +reading the element back out is what makes the shape: `self.t = (self.t,)` nests instead, and +settles on its own. + +the signatures are left uninferred because an inferred one reaches the attribute through the +method's return type as well, and a second route into the cycle changes which query is its head — +the shape under test is the one the attribute makes on its own: + +```toml +[analysis] +infer-unannotated-signatures = false +``` + +```py +class Subscript: + def g(self): + self.t = (1,) + + def f(self): + self.t = (self.t[0],) + reveal_type(self.t) # revealed: tuple[Divergent] + +class Unpacked: + def g(self): + self.t = (1,) + + def f(self): + (a,) = self.t + self.t = (a,) + reveal_type(self.t) # revealed: tuple[Divergent] + +class Starred: + def g(self): + self.t = (1,) + + def f(self): + self.t = (*self.t,) + reveal_type(self.t) # revealed: tuple[Divergent, ...] + +class Nested: + def g(self): + self.t = ((1,),) + + def f(self): + self.t = ((self.t[0][0],),) + reveal_type(self.t) # revealed: tuple[tuple[Divergent]] +``` + +## a type context that is the cycle's own marker + +two attributes each rebuilt out of the other's elements reach a fixed point in a handful of rounds — +and then keep going. the query an annotated inference runs under is interned on the expected type, +so each round's expected type interns a key of its own, and each key brings a fresh divergence +marker named after it. the marker names the query, the query is named by the key, and the key holds +the marker, so the round count is the only thing still moving. + +what the marker says is that the cycle has not reached a type yet, which is no guidance, so a +context holding one is dropped and the expression is inferred bare — the same bare key every round: + +```py +class Mutual: + def g(self): + self.a = (1,) + self.b = (2,) + + def f(self): + self.a = (self.b[0],) + self.b = (self.a[0],) + reveal_type(self.a) # revealed: tuple[Divergent] + reveal_type(self.b) # revealed: tuple[Divergent] +``` + +## a method that hands back its own bound method + +`return self.dispatch` makes `dispatch` return a bound method of `dispatch`, so the type is a cycle +rather than a tree. every walk over it — expanding a signature, comparing two of them, writing one +into a message — arrives back where it started, and each stops at the second visit instead of +following the cycle until the stack runs out. + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def dispatch(self, frame): + return self.dispatch + + def use(self): + # revealed: bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(...) + reveal_type(self.dispatch) + + # a call rebinds the receiver, which maps the signature the cyclic return type sits in + # revealed: bound method Self@use.dispatch(frame: Literal[1]) -> bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(...) + reveal_type(self.dispatch(1)) +``` + +## a self-referential bound method written into a diagnostic + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def dispatch(self, frame): + return self.dispatch + + def use(self) -> int: + return self.dispatch # error: [invalid-return-type] +``` + +## two methods that hand back each other's bound method + +each comparison of the cyclic return type against itself asks for a copy of one signature freshened +past the other, so the next round needs one nonce more than the last. no two rounds are ever equal, +which leaves nothing for a memo or a cycle guard to close on — the freshening is bounded instead, +and a pair that runs past the bound is refused rather than pursued. + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def trace(self, event): + if event: + return self.exception(event) + return self.trace + + def exception(self, event): + return self.trace + + def use(self): + # revealed: bound method Self@use.exception(event) -> bound method Self@use.trace(event) -> (bound method Self@use.trace(event) -> Divergent) | (bound method Self@use.trace(...)) + reveal_type(self.exception) +``` + +## a tuple grown in a loop, widened while a cycle is being recovered + +`args` gains an element every time round the loop, so no round of the fixed-point iteration ever +repeats the one before it. cycle recovery gives the lengths up — nothing in the program says where +the growth stops — and keeps the element type, which the program really does determine. + +giving them up means unioning the element types together, and the ordinary union builder simplifies +its elements against one another. those simplifications are relation checks, and a relation check on +a type variable standing for an unannotated parameter answers what that parameter's bound is by +inferring the whole enclosing body — a query the cycle being recovered is already running. salsa +rejects a recovery function that acquires a cycle head of its own, so recovery builds this union +without the simplification. + +```py +def f(sequence, **kw): + args = (sequence,) + for k in kw: + args = args + (k,) + + reveal_type(args) # revealed: tuple[sequence@f | str, ...] +``` + +## an `assert` whose narrowing takes away the reason it narrowed + +what a body requires of an unannotated parameter is read off the body, and the body is then checked +against what that reading produced, so the two are settled by running them against each other until +they agree. + +`assert isinstance(proto, int) and proto <= 5` says `proto` has to be an `int`. once that is +`proto`'s bound, though, `isinstance(proto, int)` is statically true — and an arm of an `and` that +is always true says nothing about which branch this is, so it is dropped and its narrowing goes with +it. the round after that has nothing to say about `proto`, which puts the bound back where it +started and lets the round after that find the narrowing again. neither round repeats the one before +it. + +what the body requires is a fact about the body, so a requirement one round found is not taken away +by a round that cannot find it. + +the signature is read from inside another function: reading it from the module asks for it before +anything has been inferred, which is not the order that reaches the cycle at all. + +```py +def opcode(stack, proto): + len(stack) + assert isinstance(proto, int) and proto <= 5 + +def check(): + reveal_type(opcode) # revealed: def opcode(stack: some Sized, proto: some int) +``` + +## a float literal a loop recomputes from itself + +basedpython folds arithmetic on float literals, so `t` is a different literal on every pass of the +loop: `0.1`, then `0.2`, then `0.4`. The type of a name bound in a loop is the union of every value +that reaches it, and each round of the fixed-point iteration adds the value the round before it +produced, so no round ever repeats the one before it. + +The union builder already stands a group of literals down to their instance type once a union +defined in terms of itself holds more of them than the fixed point can afford — which is how a loop +counting `int` literals settles on `int`. Float and complex literals have no such group, so nothing +bounded them. + +```by +def run(): + t = 0.1 + while True: + t = t * 2 + reveal_type(t) # revealed: float +``` + +## a float literal a loop recomputes from itself through a generic call + +The same growth reaches the union a generic call's solve builds, which is assembled separately from +the one that approximates the loop. `min` hands back whatever its arguments have in common, so the +literal the last pass produced comes back out of the call and goes round again. + +```by +def run(): + t = 0.1 + while True: + t = min(0.1, t * 2) + reveal_type(t) # revealed: float +``` + +## a complex literal a loop recomputes from itself + +Complex literals fold the same way and are held the same way, so they need the same bound. + +```by +def run(): + t = 1j + while True: + t = t * 2 + reveal_type(t) # revealed: complex +``` + +## float literals a loop does not recompute + +Nothing is given up when the values a loop binds do not depend on the ones it bound before, however +many of them there are. + +```by +def run(flag: int): + for _ in range(3): + if flag == 0: + t = 0.1 + elif flag == 1: + t = 0.2 + elif flag == 2: + t = 0.3 + elif flag == 3: + t = 0.4 + elif flag == 4: + t = 0.5 + else: + t = 0.6 + reveal_type(t) # revealed: 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 +``` diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 04a0bd56c1..e99c2868f4 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -1905,9 +1905,14 @@ impl<'db> Type<'db> { let unioned = UnionType::from_elements_cycle_recovery(db, env, [previous, self]); unioned.collapse_tuple_lengths(db, env) } - .recursive_type_normalized_impl_with_cycle(db, env, cycle) + // the widening comes before the normalizer, never after it: the tuple it builds has an + // element unioned out of the members it replaced, and an element mentioning the cycle's own + // marker is exactly what the normalizer folds back onto that marker. widening last would + // hand back a type the normalizer never saw, the next round would normalize it, and the + // widened and marked forms would alternate without either being reached .without_growing_tuple_lengths(db, env, previous) .without_growing_self_nesting(db, env, previous, cycle) + .recursive_type_normalized_impl_with_cycle(db, env, cycle) } /// basedpython: `self`, with a class nested inside itself that the fixed-point iteration @@ -2107,7 +2112,9 @@ impl<'db> Type<'db> { let mut elements = Vec::new(); for element in self.union_elements(db) { match element.exact_tuple_instance_spec(db) { - Some(spec) => elements.push(spec.homogeneous_element_type(db, env)), + Some(spec) => { + elements.push(spec.homogeneous_element_type_in_cycle_recovery(db, env)); + } None => rest.push(element), } } @@ -2157,15 +2164,17 @@ impl<'db> Type<'db> { .and_then(|instance| instance.tuple_spec(db, env)) .filter(|spec| spec.as_fixed_length().is_some()) { - Some(spec) => element_types.push(spec.homogeneous_element_type(db, env)), + Some(spec) => { + element_types.push(spec.homogeneous_element_type_in_cycle_recovery(db, env)); + } None => kept.push(*element), } } - let element = UnionType::from_elements(db, env, element_types); + let element = UnionType::from_elements_cycle_recovery(db, env, element_types); kept.push(Type::tuple(Some( crate::types::tuple::TupleType::homogeneous(db, env, element), ))); - UnionType::from_elements(db, env, kept) + UnionType::from_elements_cycle_recovery(db, env, kept) } pub(crate) fn is_deeply_nested(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> bool { @@ -8770,28 +8779,23 @@ impl<'db> Type<'db> { deferred.re_evaluate(db, env, operands) } - Type::FunctionLiteral(function) => visitor.visit(db, self, type_mapping, || { + Type::FunctionLiteral(function) => { + let mapped = Type::FunctionLiteral(function.apply_type_mapping_impl( + db, + type_mapping, + tcx, + visitor, + )); match type_mapping { // Promote the types within the signature before promoting the signature to its // callable form. TypeMapping::Promote( PromotionMode::On, PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, - ) => Type::FunctionLiteral(function.apply_type_mapping_impl( - db, - type_mapping, - tcx, - visitor, - )) - .promote_impl(db, visitor.env), - _ => Type::FunctionLiteral(function.apply_type_mapping_impl( - db, - type_mapping, - tcx, - visitor, - )), + ) => mapped.promote_impl(db, visitor.env), + _ => mapped, } - }), + } Type::BoundMethod(method) => Type::BoundMethod(BoundMethodType::new( db, diff --git a/crates/ty_python_semantic/src/types/constraints.rs b/crates/ty_python_semantic/src/types/constraints.rs index 50f7c6fc8a..9246a0b9ef 100644 --- a/crates/ty_python_semantic/src/types/constraints.rs +++ b/crates/ty_python_semantic/src/types/constraints.rs @@ -2079,7 +2079,7 @@ impl ConstraintId { /// /// Atomic types and bare typevars have constructor depth zero. The typevar depth is `0` if `ty` /// does not contain any typevars. -fn max_constructor_and_typevar_depth<'db>( +pub(crate) fn max_constructor_and_typevar_depth<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 0a394696ee..e29bc46119 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -7220,9 +7220,16 @@ pub(super) fn refutable_unpacking_applies<'db>( // length is not worth complaining about when the contents were never stated either. // `Any` is a different matter: it is what someone writes to say the contents are // anything, and `list[Any]` states the length just as precisely as `list[int]` does + // + // the divergence marker is the same case at its strongest. a tuple grown out of its own + // value — `self.t = (a,)` where `a` came from unpacking `self.t` — is widened to + // `tuple[Divergent, ...]` by cycle recovery, because the iteration had no fixed point to + // reach. that variable length is the recovery's own artefact, not something the program + // says, so a `may not have exactly one element` here would be complaining about our + // having given up rather than about the code if value_tuple .variable_element_type(db) - .is_some_and(|element| element.is_unknown()) + .is_some_and(|element| element.is_unknown() || element.is_divergent()) { return false; } diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 533de9d5d3..9d62db58c7 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -204,6 +204,30 @@ impl<'db> DisplaySettings<'db> { } } + /// Begin displaying the signature of `function`, or `None` when doing so would recurse. + /// + /// A function's signature can name the function itself: an inferred return type of + /// `self.f` makes `f` return a callable over `f`. Rendering that nests forever, so every + /// site that writes a signature belonging to a `FunctionType` must go through here — the + /// exhausted result is a truncated `(...)`. The depth limit catches the case where the + /// nested function is an equal-but-distinct value, as it is once a signature has been + /// rebound to a receiver. + #[must_use] + fn enter_function(&self, function: FunctionType<'db>) -> Option { + const MAX_FUNCTION_TYPE_DISPLAY_DEPTH: usize = 4; + if self.visited_function_types.contains(&function) + || self.visited_function_types.len() >= MAX_FUNCTION_TYPE_DISPLAY_DEPTH + { + return None; + } + let mut visited = (*self.visited_function_types).clone(); + visited.insert(function); + Some(Self { + visited_function_types: Rc::new(visited), + ..self.clone() + }) + } + #[must_use] pub(crate) fn preserve_long_unions(self) -> Self { Self { @@ -1701,9 +1725,29 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { .display_with(db, self.env, self.settings.clone()) .fmt_detailed(f), Type::BoundMethod(bound_method) => { - let function = bound_method.function(db); - let self_ty = bound_method.self_instance(db); - let bound_signatures = bound_method.bound_signatures(db); + let function = bound_method.function(self.db); + let self_ty = bound_method.self_instance(self.db); + + let write_prefix = |f: &mut TypeWriter<'_, '_, 'db>| { + f.set_invalid_type_annotation(); + f.write_str("bound method ")?; + DisplayMaybeParenthesizedType { + ty: self_ty, + db: self.db, + env: self.env, + settings: self.settings.singleline(), + } + .fmt_detailed(f)?; + f.write_char('.')?; + f.with_type(self.ty).write_str(function.name(self.db)) + }; + + let Some(settings) = self.settings.enter_function(function) else { + write_prefix(f)?; + return f.write_str("(...)"); + }; + + let bound_signatures = bound_method.bound_signatures(self.db); match bound_signatures.overloads.as_slice() { [signature] => { @@ -1711,50 +1755,38 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { signature.should_hide_self_from_display(db, self.env); let type_parameters = DisplayOptionalGenericContext { generic_context: signature.generic_context.as_ref(), - db, + db: self.db, hide_unused_self, }; - f.set_invalid_type_annotation(); - f.write_str("bound method ")?; - DisplayMaybeParenthesizedType { - ty: self_ty, - db, - env: self.env, - settings: self.settings.singleline(), - } - .fmt_detailed(f)?; - f.write_char('.')?; - f.with_type(self.ty).write_str(function.name(db))?; + write_prefix(f)?; type_parameters.fmt_detailed(f)?; signature .display_with( self.db, - env, - self.settings - .disallow_signature_name() - .name_already_written(), + self.env, + settings.disallow_signature_name().name_already_written(), ) .fmt_detailed(f) } signatures => { // TODO: How to display overloads? - if !self.settings.multiline { + if !settings.multiline { // TODO: This should ideally have a TypeDetail but we actually // don't have a type for @overload (we just detect the decorator) f.write_str("Overload")?; f.write_char('[')?; } - let separator = if self.settings.multiline { "\n" } else { ", " }; + let separator = if settings.multiline { "\n" } else { ", " }; let mut join = f.join(separator); for signature in signatures { join.entry(&signature.display_with( - db, + self.db, self.env, - self.settings.clone(), + settings.clone(), )); } join.finish()?; - if !self.settings.multiline { + if !settings.multiline { f.write_str("]")?; } Ok(()) @@ -2511,25 +2543,14 @@ struct DisplayFunctionType<'env, 'db> { impl<'db> FmtDetailed<'db> for DisplayFunctionType<'_, 'db> { fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result { - // Detect self-referential function types to prevent infinite recursion, - // and limit display depth for chains of different function types - // (e.g. multiple redefinitions with `TypeOf[foo]` return types). - const MAX_FUNCTION_TYPE_DISPLAY_DEPTH: usize = 4; let env = self.env; let db = self.db; - if self.settings.visited_function_types.contains(&self.ty) - || self.settings.visited_function_types.len() >= MAX_FUNCTION_TYPE_DISPLAY_DEPTH - { + let Some(settings) = self.settings.enter_function(self.ty) else { f.set_invalid_type_annotation(); f.write_str("def ")?; write!(f, "{}", self.ty.name(db))?; return f.write_str("(...)"); - } - - let mut settings = self.settings.clone(); - let mut visited = (*settings.visited_function_types).clone(); - visited.insert(self.ty); - settings.visited_function_types = Rc::new(visited); + }; let signature = self.ty.signature(db); diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index be2c5fee6d..cadf56e695 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1996,12 +1996,37 @@ impl<'db> FunctionType<'db> { ) } + /// An inferred signature can mention the function it belongs to: `def f(self): return + /// self.f` gives `f` a return type that contains a bound method of `f`. Mapping that + /// signature reaches the same function again, so the descent has to be guarded here rather + /// than at each of the [`Type`] variants that hold a `FunctionType`. pub(crate) fn apply_type_mapping_impl<'a>( self, db: &'db dyn Db, type_mapping: &TypeMapping<'a, 'db>, tcx: TypeContext<'db>, visitor: &ApplyTypeMappingVisitor<'_, 'db>, + ) -> Self { + let mapped = visitor.visit(db, Type::FunctionLiteral(self), type_mapping, || { + Type::FunctionLiteral(self.apply_type_mapping_to_signatures( + db, + type_mapping, + tcx, + visitor, + )) + }); + let Type::FunctionLiteral(mapped) = mapped else { + return self; + }; + mapped + } + + fn apply_type_mapping_to_signatures<'a>( + self, + db: &'db dyn Db, + type_mapping: &TypeMapping<'a, 'db>, + tcx: TypeContext<'db>, + visitor: &ApplyTypeMappingVisitor<'_, 'db>, ) -> Self { // Returned-callable rescoping and type-alias specialization should not rebuild signatures from the // function literal; doing so can re-enter recursive `TypeOf` evaluation. diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index e3b8c6d2bd..ba47f76812 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -610,7 +610,7 @@ impl<'db> InferExpression<'db> { expression: Expression<'db>, tcx: TypeContext<'db>, ) -> InferExpression<'db> { - if tcx.annotation().is_some() { + if tcx.is_worth_interning(db, &ProgramEnvironment::from_scope(expression.scope(db))) { InferExpression::WithContext(ExpressionWithContext::new(db, expression, tcx)) } else { InferExpression::Bare(expression) @@ -645,7 +645,7 @@ pub(super) struct ScopeWithContext<'db> { impl<'db> InferScope<'db> { fn new(db: &'db dyn Db, scope: ScopeId<'db>, tcx: TypeContext<'db>) -> InferScope<'db> { - if tcx.annotation().is_some() { + if tcx.is_worth_interning(db, &ProgramEnvironment::from_scope(scope)) { InferScope::WithContext(ScopeWithContext::new(db, scope, tcx)) } else { InferScope::Bare(scope) @@ -729,6 +729,24 @@ impl<'db> TypeContext<'db> { self.target } + /// basedpython: whether this context earns a query key of its own + /// + /// the key an annotated inference runs under is *interned*, so a context that differs from the + /// one the last round used is a different query, carrying a `Type::divergent` initial value + /// named after it. a context holding a divergence marker closes that into a loop: the marker is + /// named after the query, the query is named by the key, and the key holds the marker — so a + /// cycle interns one more key every round and never reaches a fixed point, however settled the + /// types it is computing already are + /// + /// a marker is the cycle's stand-in for a type it has not reached yet, which is no guidance at + /// all, so the context is dropped and the expression is inferred bare. the next round drops the + /// same context and asks for the same bare key, unchanged + fn is_worth_interning(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> bool { + self.annotation().is_some_and(|annotation| { + !any_over_type(db, env, annotation, false, |ty| ty.is_divergent()) + }) + } + /// basedpython: the expected type a bare name in this context resolves /// against — the annotation, or the result type of the call this expression /// is the callee of diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index 97e95e4a3c..d5c8a1ad58 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -34,7 +34,7 @@ use crate::reachability::ReachabilityConstraintsExtension; use crate::types::ProgramEnvironment; use crate::types::call::CallArguments; use crate::types::callable::CallableType; -use crate::types::constraints::ConstraintSetBuilder; +use crate::types::constraints::{ConstraintSetBuilder, max_constructor_and_typevar_depth}; use crate::types::function::OverloadLiteral; use crate::types::narrow::{NarrowingConstraint, infer_narrowing_constraints}; use crate::types::protocol_class::InlineProtocolMember; @@ -64,7 +64,7 @@ use crate::types::{ cycle_initial = |_, id, _| Type::divergent(id), cycle_fn = |db, cycle, previous: &Type<'db>, value: Type<'db>, overload: OverloadLiteral<'db>| { let env = &ProgramEnvironment::from_file(overload.program_file(db)); - value.cycle_normalized(db, env, *previous, cycle) + divergence_bounded(db, env, value, cycle).cycle_normalized(db, env, *previous, cycle) }, heap_size = ruff_memory_usage::heap_size, )] @@ -91,6 +91,41 @@ pub(crate) fn inferred_return_type<'db>( ) } +/// The deepest a recovered return type may nest before the recursion that built it +/// is called what it is. +/// +/// A hand-written return type is a constructor or two deep — `list[int]`, +/// `dict[str, list[int]]`. Anything far past that inside a cycle was assembled one +/// layer per iteration rather than written by anybody. +const RETURN_TYPE_NESTING_LIMIT: u16 = 8; + +/// `value` with a return type that grows a constructor deeper every iteration +/// replaced by the divergence marker it already stands for. +/// +/// A body that returns a call taking the function itself — `def g(n): return map(g, n)` +/// — has no return type to reach: it is `map[map[…]]` without end. Ordinarily +/// [`Type::cycle_normalized`] folds such a type back onto the marker the cycle started +/// from, but the marker only survives while the type is *built*; passing through a +/// generic call's solve leaves a concrete type behind with nothing left to fold on, and +/// the fixed point recedes by one constructor per iteration forever. +/// +/// So bound the nesting rather than the iterations: past the bound the value is +/// replaced by the cycle head's own `Divergent`, which is what the marker-preserving +/// path would have produced, and the next iteration reproduces it unchanged. The bound +/// reads only the value and the cycle, so the query stays a function of its inputs. +fn divergence_bounded<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + value: Type<'db>, + cycle: &salsa::Cycle, +) -> Type<'db> { + let (constructor_depth, _) = max_constructor_and_typevar_depth(db, env, value); + if constructor_depth < RETURN_TYPE_NESTING_LIMIT { + return value; + } + cycle.head_ids().next().map_or(value, Type::divergent) +} + /// The return type `node`'s body determines, given what its expressions were inferred as. /// /// [`inferred_return_type`] reads those out of a completed scope inference. The @@ -314,9 +349,19 @@ pub(crate) fn inferred_parameter_bound<'db>( /// /// Every parameter is answered in one pass, because the expensive half — inferring /// the body, and re-binding each call in it — is shared between them. +/// +/// This reads the body, and the body is checked against the bounds this produces, so +/// the answer is reached by iterating the two to a fixed point. Each round is allowed +/// to say more about a parameter than the round before it, and to say something +/// different; the one thing it may not do is stop answering for a parameter it has +/// already answered for, which is what [`ParameterConstraints::keeping_requirements_seen`] +/// enforces. #[salsa::tracked( returns(ref), cycle_initial = |_, _, _| ParameterConstraints::default(), + cycle_fn = |_, _, previous: &ParameterConstraints<'db>, value: ParameterConstraints<'db>, _| { + value.keeping_requirements_seen(previous) + }, heap_size = ruff_memory_usage::heap_size, )] pub(crate) fn body_parameter_constraints<'db>( @@ -348,8 +393,8 @@ pub(crate) fn body_parameter_constraints<'db>( // a name bound more than once cannot stand for one value: which of them a later use is // about is not a question this can answer - let single_bindings = index - .place_table(body_scope.file_scope_id(db)) + let place_table = index.place_table(body_scope.file_scope_id(db)); + let single_bindings: FxHashSet = place_table .symbols() .filter(|symbol| symbol.is_bound() && !symbol.is_reassigned()) .map(|symbol| symbol.name().clone()) @@ -386,13 +431,26 @@ pub(crate) fn body_parameter_constraints<'db>( entries.extend(asserted_parameter_types(db, env, index, body_scope, node)); // a parameter a nested scope captured keeps nothing: that body is checked against this - // bound, and this walk never saw what it does with the name - if !collector.captured.is_empty() { - entries.retain(|(parameter, _)| { - parameter_definition_name(db, *parameter) - .is_none_or(|name| !collector.captured.contains(&name)) - }); - } + // bound, and this walk never saw what it does with the name. + // + // a parameter its own body rebinds keeps nothing either, for the same reason a rebound + // local does. after + // + // while tb.tb_next: + // tb = tb.tb_next + // + // the name stands for whatever the rebinding produced, not for what the caller passed, so + // the uses below it are no requirement on the argument — and bounding the argument by them + // anyway makes the body fail against its own bound, because the rebinding lands on the + // member type the bound itself invented + entries.retain(|(parameter, _)| { + parameter_definition_name(db, *parameter).is_none_or(|name| { + !collector.captured.contains(&name) + && place_table + .symbol_id(name.as_str()) + .is_some_and(|symbol| !place_table.symbol(symbol).is_reassigned()) + }) + }); entries.sort_by_key(|(parameter, _)| *parameter); @@ -417,6 +475,43 @@ impl<'db> ParameterConstraints<'db> { .map(|(_, ty)| *ty) .collect() } + + /// This round's requirements, plus those of every parameter this round stopped + /// answering for. + /// + /// What a body requires of a parameter is a fact about the body, so a requirement one + /// round of the cycle found does not stop holding because a later round could not find + /// it. A round really can lose one. `assert isinstance(x, int) and x <= 5` narrows `x` + /// to `int` only while `isinstance(x, int)` can still come out false; the round after + /// that narrowing has become `x`'s bound the test is statically true, and an `and` arm + /// that is always true says nothing about which branch this is, so it is dropped — + /// taking its narrowing with it. That puts the bound back where it started, and the + /// round after finds the narrowing again. Neither round repeats the one before it, and + /// the iteration has no fixed point to reach. + /// + /// Requirements only ever being added is what leaves it one. + fn keeping_requirements_seen(mut self, previous: &Self) -> Self { + let dropped: Vec<_> = previous + .entries + .iter() + .filter(|(parameter, _)| { + !self + .entries + .iter() + .any(|(answered, _)| answered == parameter) + }) + .copied() + .collect(); + if dropped.is_empty() { + return self; + } + + let mut entries = self.entries.into_vec(); + entries.extend(dropped); + entries.sort_by_key(|(parameter, _)| *parameter); + self.entries = entries.into_boxed_slice(); + self + } } /// Feed each `assert` at the top level of the body back into the value it is about. diff --git a/crates/ty_python_semantic/src/types/relation.rs b/crates/ty_python_semantic/src/types/relation.rs index 17741030a1..8e9b5b4654 100644 --- a/crates/ty_python_semantic/src/types/relation.rs +++ b/crates/ty_python_semantic/src/types/relation.rs @@ -1251,6 +1251,17 @@ impl<'a, 'c, 'db> TypeRelationChecker<'a, 'c, 'db> { self.never() } + /// The answer for a pair of signatures whose comparison reproduces itself at an ever + /// greater typevar freshness. + /// + /// Such a pair is undecidable for the same reason a recursively-specialized structural + /// type is, and gets the same conservative answer — see + /// [`Self::recursive_type_pair_fallback`]. Rejecting is what keeps both members of a + /// self-referential union rather than folding one into the other. + pub(super) fn diverged_signature_pair(&self) -> ConstraintSet<'db, 'c> { + self.never() + } + /// Is `target` a metaclass instance (a nominal instance of a subclass of `builtins.type`)? /// /// This does not include all types that are subtypes of `builtins.type`! The semantic diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index d80b97d412..e7572099c2 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -1025,7 +1025,11 @@ impl<'db> UnionBuilder<'db> { self.elements.swap_remove(index); } } - _ => self.push_type(ty, seen_aliases), + _ => { + if !self.widen_ungrouped_literals(literal, seen_aliases) { + self.push_type(ty, seen_aliases); + } + } } } // Adding `object` to a union results in `object`. @@ -1034,6 +1038,71 @@ impl<'db> UnionBuilder<'db> { } } + /// basedpython: stand a recursively defined union's float and complex literals down to + /// their instance type once there are more of them than the fixed point can afford. + /// + /// Every literal kind handled above lives in a [`UnionElement`] group of its own, and + /// widening that group is what stops a union defined in terms of itself from gaining an + /// element on every round of the fixed-point iteration. Float and complex literals — + /// which only basedpython has, and only basedpython does arithmetic on — are kept as + /// ordinary union elements, so nothing bounded them. A loop that computes the next value + /// from the last, `t = t * 2`, folds `0.1 | 0.2 | 0.4 | …` one element longer each round, + /// so no round repeats the one before it and the iteration never converges. + /// + /// The one type a loop like that really determines is `float`, so past the limit that is + /// what the literals stand for. Both places a growing union is assembled are covered: the + /// loop-header union that approximates a loop's fixed point, and the union a cycle + /// recovery function rebuilds. + /// + /// The rule is written for every literal kind without a group rather than for these two, + /// so a kind added later is bounded whether or not anybody remembers this. The kinds that + /// cannot proliferate — a `bool`, a `LiteralString` — never reach the limit anyway. + /// + /// Returns `true` when the literal is already accounted for and the caller must not add + /// it. + fn widen_ungrouped_literals( + &mut self, + literal: LiteralValueType<'db>, + seen_aliases: &mut Vec>, + ) -> bool { + // A union not defined in terms of itself is complete as soon as it is built, so there + // is nothing to converge and no reason to give its literals up. + if !self.recursively_defined.is_yes() { + return false; + } + + let db = self.db; + let fallback = literal.fallback_instance(db, &self.env); + let mut same_kind = SmallVec::<[usize; 8]>::new(); + for (index, element) in self.elements.iter().enumerate() { + let UnionElement::Type(existing) = element else { + continue; + }; + // Once widened, the instance type stands for every literal of its kind, this one + // included. Outside recovery `push_type` reaches the same conclusion through the + // ordinary redundancy check, which also applies the simplifications skipped here. + if self.cycle_recovery && *existing == fallback { + return true; + } + if let Type::LiteralValue(existing_literal) = existing + && existing_literal.fallback_instance(db, &self.env) == fallback + { + same_kind.push(index); + } + } + + if same_kind.len() < MAX_RECURSIVE_UNION_LITERALS { + return false; + } + + // Removing from the back leaves the earlier indices where they were. + for index in same_kind.into_iter().rev() { + self.elements.remove(index); + } + self.add_in_place_impl(fallback, seen_aliases); + true + } + fn push_type(&mut self, ty: Type<'db>, seen_aliases: &mut Vec>) { let env = &self.env; let db = self.db; @@ -2084,6 +2153,60 @@ mod tests { assert_eq!(union.build(), KnownClass::Int.to_instance(db, &env)); + // basedpython: float literals have no `UnionElement` group of their own, so they are + // bounded by `widen_ungrouped_literals` instead. A loop doubling a float — the shape + // that first ran the fixed point out of iterations — reaches the limit this way. One + // value past the limit, since it takes that many to widen. + let doubling = || { + (0..=MAX_RECURSIVE_UNION_LITERALS).scan(0.1, |value, _| { + let doubled = *value; + *value *= 2.0; + Some(doubled) + }) + }; + let over_limit = doubling().map(Type::float_literal).collect::>(); + + let float_union = over_limit.iter().copied().fold( + UnionBuilder::new(db, &env) + .cycle_recovery(true) + .recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!(float_union.build(), KnownClass::Float.to_instance(db, &env)); + + // The loop-header union that approximates a loop's fixed point is not built in + // recovery mode, and it grows the same way. + let float_loop_union = over_limit.iter().copied().fold( + UnionBuilder::new(db, &env).recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!( + float_loop_union.build(), + KnownClass::Float.to_instance(db, &env) + ); + + let complex_union = doubling() + .map(|value| Type::complex_literal(db, 0.0, value)) + .fold( + UnionBuilder::new(db, &env) + .cycle_recovery(true) + .recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!( + complex_union.build(), + KnownClass::Complex.to_instance(db, &env) + ); + + // A union nothing defines in terms of itself settles in one go, so its literals are + // kept however many there are. + let non_recursive = over_limit + .iter() + .copied() + .fold(UnionBuilder::new(db, &env), UnionBuilder::add) + .build(); + assert_ne!(non_recursive, KnownClass::Float.to_instance(db, &env)); + let assert_widens = |literal, instance| { for (first, second) in [(literal, instance), (instance, literal)] { let union = UnionBuilder::new(db, &env) diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 0411d8a1c8..509b3ab6ae 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -42,7 +42,8 @@ use crate::types::relation::{ use crate::types::tuple::{Tuple, TupleType, VariableSegment}; use crate::types::typed_dict::extract_unpacked_typed_dict_keys_from_kwargs_annotation; use crate::types::typevar::{ - TypeVarInstance, TypeVarKind, TypeVarSet, max_typevar_freshness_matching_generic_context, + MAX_TYPEVAR_FRESHNESS_DELTA, TypeVarInstance, TypeVarKind, TypeVarSet, + max_typevar_freshness_matching_generic_context, }; use crate::types::{ ApplyTypeMappingVisitor, BindingContext, BoundTypeVarIdentity, BoundTypeVarInstance, @@ -2683,6 +2684,9 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { .max_typevar_freshness_matching_generic_context(db, generic_context) .map(|freshness| freshness.increment().value()) { + if delta > MAX_TYPEVAR_FRESHNESS_DELTA { + return self.diverged_signature_pair(); + } freshened_source = source.freshen_bound_typevars(db, env, delta); &freshened_source } else { @@ -2695,6 +2699,9 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { .max_typevar_freshness_matching_generic_context(db, generic_context) .map(|freshness| freshness.increment().value()) { + if delta > MAX_TYPEVAR_FRESHNESS_DELTA { + return self.diverged_signature_pair(); + } freshened_target = target.freshen_bound_typevars(db, env, delta); &freshened_target } else { diff --git a/crates/ty_python_semantic/src/types/tuple.rs b/crates/ty_python_semantic/src/types/tuple.rs index 3a9714f301..150fee371f 100644 --- a/crates/ty_python_semantic/src/types/tuple.rs +++ b/crates/ty_python_semantic/src/types/tuple.rs @@ -2410,14 +2410,49 @@ impl<'db> Tuple, VariableSegment<'db>> { db: &'db dyn Db, env: &ProgramEnvironment<'db>, ) -> Type<'db> { + self.element_types_unioned(db, env, false) + } + + /// basedpython: the same answer as [`Self::homogeneous_element_type`], for a caller that is + /// itself part of a cycle recovery function. + /// + /// The ordinary union builder simplifies its elements against one another, and a relation + /// check can force arbitrary inference — a type variable standing for an unannotated + /// parameter answers what its bound is by inferring the whole enclosing function body. A + /// recovery function that reached a query already on the stack would acquire a cycle head of + /// its own, which salsa rejects outright, so recovery builds its unions without that + /// simplification. + pub(crate) fn homogeneous_element_type_in_cycle_recovery( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Type<'db> { + self.element_types_unioned(db, env, true) + } + + fn element_types_unioned( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + cycle_recovery: bool, + ) -> Type<'db> { + // cycle recovery mode already leaves aliases packed, so the two modes agree about aliases + let mut builder = UnionBuilder::new(db, env) + .unpack_aliases(false) + .cycle_recovery(cycle_recovery); match self { Tuple::Fixed(tuple) => { - UnionType::from_elements_leave_aliases(db, env, tuple.iter_all_elements()) + for element in tuple.iter_all_elements() { + builder.add_in_place(element); + } } Tuple::Variable(tuple) => { - UnionType::from_elements_leave_aliases(db, env, tuple.iter_all_elements(db)) + for element in tuple.iter_all_elements(db) { + builder.add_in_place(element); + } } } + builder.build() } fn tuple_class_type(&self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> Type<'db> { diff --git a/crates/ty_python_semantic/src/types/typevar.rs b/crates/ty_python_semantic/src/types/typevar.rs index 96c128a8ad..fab5cb0f91 100644 --- a/crates/ty_python_semantic/src/types/typevar.rs +++ b/crates/ty_python_semantic/src/types/typevar.rs @@ -1044,6 +1044,16 @@ pub struct TypeVarNonce(u32); // This type does not have any heap storage. impl get_size2::GetSize for TypeVarNonce {} +/// How far a signature's typevars may be freshened past the signature it is compared against. +/// +/// Freshening only has to lift one signature's typevars clear of the other's, so the distance +/// needed is the nesting of same-context generic signatures inside one comparison — a handful +/// at most in real code. It is unbounded only when a comparison reproduces itself at a greater +/// freshness, which a self-referential inferred return type does: `def f(self): return self.f` +/// makes each comparison of `f` against itself demand a signature one nonce fresher than the +/// last, so no two rounds are ever equal and no memo or cycle guard can close the loop. +pub(crate) const MAX_TYPEVAR_FRESHNESS_DELTA: u32 = 32; + impl TypeVarNonce { pub(crate) const NONE: Self = Self(0); const FIRST: Self = Self(1); diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 5fe04e062b..9d5f748813 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -105,6 +105,12 @@ by compile --no-any # refuse to leave a gradual-typed function inter by compile --require-native # refuse to leave *any* function interpreted ``` +the output directory mirrors the *module* tree, the way `by build`'s does: the +package member `pkg/sub/dup.py` lands at `out/pkg/sub/dup.cpython-313-darwin.so`, +and the package `pkg/sub/__init__.py` at +`out/pkg/sub/__init__.cpython-313-darwin.so` — so `out/` can go on `sys.path` as +it stands and every module imports under the dotted name it was compiled as + `--no-any` buys no speed on its own — it is a **predictability contract**. a gradual type is the commonest reason a function silently stays interpreted, and a decline is invisible unless you look for it, so a module that means to be fully diff --git a/docs/basedpython/development/compilation/benchmarks.md b/docs/basedpython/development/compilation/benchmarks.md new file mode 100644 index 0000000000..dc2185c9a9 --- /dev/null +++ b/docs/basedpython/development/compilation/benchmarks.md @@ -0,0 +1,203 @@ +# the benchmark suite + +```sh +cargo build --release --bin by +uv run --no-project --python 3.13 python scripts/native-bench/bench.py +``` + +the suite lives in `scripts/native-bench/`: `bench.py` stages and builds, +`timer.py` times, `programs/` holds the benchmarks and `programs.toml` says what +each is for and what it is allowed to leave interpreted + +it needs a **release** build — a debug `by` is too slow to measure anything with +— and it needs `uv`, which fetches both the interpreter and mypyc + +## the one thing to know + +**the ratio is the measurement.** absolute times on this suite move by a factor +of four with machine load: cpython's own `dot` has read 47ms and 209ms on the +same laptop in the same week. no number here is reported from one build's clock +alone + +and **every row carries its own error bar**. each benchmark is compiled *twice*, +independently, by the same compiler, and both builds are timed. two builds of one +program are one program — literally: the generated C is byte-identical once the +module name is normalised out — so whatever ratio the suite reports between them +is noise it invented. that is the `noise` column, and nothing smaller than a +row's own noise is a finding + +## how a run works + +four builds of each benchmark, staged under four module names of their own: + +| build | what it is | +| --------- | ------------------------------ | +| `cpython` | the source, interpreted | +| `by` | `by compile` | +| `control` | `by compile` again, separately | +| `mypyc` | mypyc, for scale | + +the distinct names are what makes the method possible. an extension module's +init hook is found by name, so two of them cannot answer to `mandel` — under +`mandel_by` and `mandel_control` they can, and all four builds then live in one +process at once + +and because they live in one process they can be **interleaved**. a run is a +sequence of rounds, and in each round every build is timed once, in an order +that rotates so none of them is always the one paying for a cold cache. a load +spike therefore lands inside the same round as everything else, and mostly +cancels when the round is turned into a quotient + +## the statistics + +the **median**, throughout — never the mean, never the minimum + +- the mean is moved by the one round that hit a scheduler +- the minimum is an extreme-value statistic. its expectation depends on how many + rounds were run and on how quiet the machine happened to be, so a minimum is + not comparable with another minimum, which is the only thing this suite is for + +a ratio is **paired**: it is the median of the per-round quotients, not the +quotient of the two medians. dividing one build's median by another's would let +a spike that landed on only one of them through unchallenged + +the interval on each ratio is the distribution-free one for a median, taken from +the binomial tail. it is exact, assumes nothing about the shape of the noise, +needs no resampling and no random numbers — so two readings of the same data +agree. below nine rounds that interval degenerates to the range of everything +seen, so nine is a floor the harness enforces rather than a default + +one sample is many calls, and how many is calibrated per benchmark: enough that +the fastest build's sample is well clear of the clock, few enough that the +slowest build's round does not dominate the run. every build runs the same +number, so the pairing stays exact + +## what it refuses + +each of these is a way this suite has actually produced a wrong number, and each +is now a refusal rather than a warning: + +| refusal | the failure it closes | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| the artefact must be a real extension module | a broken shim made `by compile` fail silently and the previous `.so` was timed in its place | +| …under this run's own root, newer than this build | the run's root is a fresh temporary directory, so a stale one cannot be reached at all | +| an interpreted build must **not** be an extension | the mirror image, which would make the baseline a compiled one | +| every build must return cpython's answer | a build that got a different answer is not a faster build | +| the decline count must match `programs.toml` exactly | a benchmark can decline and quietly measure interpreted code while still posting a number | +| an unknown name, an unlisted program, an empty run | a harness that matches nothing looks exactly like a harness that found nothing wrong | + +the decline check is the one worth dwelling on. `by compile` never fails on code +it cannot lower — that function runs from its interpreted definition instead — so +a "compiled" number can silently be an interpreted one. `programs.toml` records +the expected count per benchmark and the run fails when it moves **in either +direction**: a compiler that started compiling something is as much a change as +one that stopped. improving the compiler edits that file in the same commit, +which makes it a readable ledger of what the backend has learned to take + +the refusals are exercised rather than asserted: + +```sh +uv run --no-project --python 3.13 python scripts/native-bench/bench.py --self-check +``` + +which builds a leg that is wrong in each of those ways and proves each one is +turned away + +## what it measures + +one axis per group, and a benchmark earns its place by being the only one on its +axis — or by being half of a pair whose *difference* is the axis: + +| group | benchmarks | the question | +| ---------- | ---------------------------------------- | ---------------------------------------------------- | +| `float` | `mandel`, `mandel_inline` | scalar float work; the pair isolates calls | +| `int` | `loops`, `bigint` | tagged integers, and leaving the word | +| `dispatch` | `calls`, `methods`, `recurse`, `inherit` | call, method, depth, and an override | +| `memory` | `alloc`, `fields`, `objects` | allocation and field access, apart and together | +| `list` | `dot`, `prefix`, `sieve` | indexed reads, growth, indexed writes | +| `tuple` | `tuples` | pack, unpack, and a two-value return | +| `set` | `sets` | membership as the whole operation | +| `dict` | `dictget`, `dicthist` | lookup-and-update, and the histogram miss | +| `str` | `words`, `chars`, `strops`, `keybuild` | build, scan, methods, and key construction | +| `boxing` | `generic`, `generic_mono` | the pair isolates what a type parameter costs | +| `control` | `excs` | a raise that is caught, and a `try` that never fires | +| `frames` | `gen`, `coro` | the two halves of the resumable-frame lowering | + +the pairs are read as pairs. `generic` alone says almost nothing; against +`generic_mono`, which is the same program with the call monomorphised by hand, +it says exactly what the type parameter costs. same for `mandel` against +`mandel_inline`, and for `objects` against `alloc`, `fields` and `methods` + +adding one is three files: the program, an entry in `programs.toml`, and a line +in the table above. the entry has to say what it measures that nothing else does + +## comparing two runs + +```sh +… bench.py --json today.json +… bench.py --baseline today.json +``` + +the quantity compared between runs is the **speedup**, not the time. absolute +times are not comparable across machines, across interpreters, or across a +Tuesday, and this suite exists because they were compared anyway + +a change is called one only when it clears the bar, and the bar is whichever is +larger: the noise the two runs measured for themselves, or +`--regression-threshold`. a regression exits non-zero, so it is detected rather +than eyeballed. a baseline from a different interpreter or a different host is +compared anyway but says so first, and a row either run found too noisy is +skipped rather than given a very wide bar + +**the noise column does not bound this.** the control is two builds in one +process at one moment, so it bounds the noise *within* a run. between two runs +there is more: a different process, a different heap layout, a different machine +mood. measured rather than assumed — two runs of an **unchanged** compiler, +forty minutes apart, agreed within 3% on 25 of 27 rows and disagreed by 8.1% and +8.4% on the other two, while both of those rows reported a ±0.8% floor for +themselves. so the default bar is 10%, which is what that evidence supports, and +not the within-run floor, which would have called both of them improvements + +the consequence is worth stating plainly: on a machine doing other work this +suite sees a 10% change and does not see a 5% one. for smaller than that, use a +machine that does nothing else, drop `--regression-threshold` to match the floor +it then reports, and confirm anything it flags by running it again + +## the half that can run anywhere + +```sh +… bench.py --verify-only +… bench.py --self-check +``` + +`--verify-only` does everything the suite does except look at a clock: it builds +all four ways, proves each artefact, checks that every build gives cpython's +answer, and checks the decline ledger. none of that depends on how busy the +machine is, so it is the half that belongs on a shared runner — and it is where +most of the *correctness* value is. it catches a benchmark that stopped +compiling, a compiled build that started answering differently, and a compiler +that quietly began declining something it used to take + +the timing half wants a machine of its own. a shared runner cannot hold a 3% +noise floor, and this suite's whole premise is that a number it cannot stand +behind should not be printed as if it could. run the table on a fixed machine, +keep the `--json`, and gate on `--baseline` against the previous one + +## reading a bad run + +a row whose noise exceeds `--noise-limit` (10% by default) is marked `!` and +does not count as measured. the run then exits non-zero even if everything else +looks fine, because a table that is *mostly* trustworthy is the kind nobody +remembers to check before quoting one of its rows + +that limit is not fastidiousness. pairing cancels the *noise* a busy machine +adds, but it does not cancel the **bias**: under preemption the longer sample +loses more, and the interpreted build's sample is the long one, so contention +pushes the reported speedup *up*. a run at load 95 on a 16-core laptop read +`fields` at 23.8x where two quiet runs both put it near 12x, and the two +supposedly identical builds disagreed by 74% in the same breath. widening the +bar does not fix a bias, so a row that noisy is skipped rather than compared + +between those two extremes the floor is simply reported. a `±6%` floor means +the run can see a 2x difference and cannot see a 10% one, which is often all +that was wanted diff --git a/docs/basedpython/development/compilation/index.md b/docs/basedpython/development/compilation/index.md index 36513b3d30..b5803f0542 100644 --- a/docs/basedpython/development/compilation/index.md +++ b/docs/basedpython/development/compilation/index.md @@ -173,4 +173,6 @@ because two of them change this design rather than extend it: what each buys the compiler, and what codegen needs their design to preserve - [runtime](runtime.md) — object model, refcounting, exceptions, interop, debugging +- [benchmarks](benchmarks.md) — what the suite measures, the method it enforces, + and what it refuses to time - [plan](plan.md) — semantic deltas, testing, milestones, risks diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 6187612e46..061f96dbb2 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -124,6 +124,38 @@ a class whose construction fell back to the interpreted definition is skipped: t `def`s the fallback ran already carry their decorators, and applying them a second time would wrap twice +### how many times a decorator runs + +python evaluates a decorator **once**, where the definition stands. the twin is +what stands there, so the twin evaluates it — and module init then evaluates the +same decorator a second time over the native definition that replaced the twin's. +the name each definition ends up bound to is right either way, so nothing shows +but the side effect: `@register` puts two entries in its registry, `@count_them` +counts one function twice + +for a module-level **function** and a module-level **class** the decorator is +therefore blanked out of the source the twin runs, so init's is the only +evaluation. blanking rather than cutting keeps every line where it was, which is +what a traceback through the twin quotes + +that leaves a window: from the twin's `def` or `class` to the moment init reaches +it, the name holds a definition nothing has decorated yet. only the module's own +body can look — everything else runs after init — so a definition whose name that +body reads **declines** rather than be compiled and decorated twice. the reads +followed are the ones the body makes as it runs, plus, transitively, everything +held behind any definition it names: `TABLE = f()` reads directly, +`def g(): return f()` called at import reads just the same. an annotation counts +only where python evaluates one, so a module with +`from __future__ import annotations` may name a class in a signature freely + +a **method's** decorator still runs twice. it is not only a side effect that +would move: the class construction itself reads what the decorator wrote, and +`ABCMeta` is the case — it computes `__abstractmethods__` from the namespace the +body left, so taking `@abstractmethod` out of the twin empties that set on every +class whose construction falls back to the interpreted definition. the answer +there is to carry the decorated method *across* from the twin rather than to +re-apply the decorator, which is not built + ### boxed classes and interpreted fallbacks a construct with no native lowering is not a compile error. the module emits its diff --git a/docs/basedpython/features/sound-types.md b/docs/basedpython/features/sound-types.md index 57d874a62d..200db76001 100644 --- a/docs/basedpython/features/sound-types.md +++ b/docs/basedpython/features/sound-types.md @@ -174,6 +174,19 @@ that a member's value was given to is read the same way: a name bound more than for one value, and a use under a narrowing is about something narrower than the value it was bound to +a parameter its own body rebinds keeps nothing at all, for the same reason. the reads above the +rebinding are not enough on their own: walking a linked structure asks only that the argument have +the member it walks along, so the rebinding lands on that member — whose value nothing described — +and the walk's next step would fail against the signature the function itself produced + +```python +def deepest(tb): + if tb.tb_next: + tb = tb.tb_next + return tb.tb_frame +# def deepest(tb) +``` + ### an `assert` at the top of the body an `assert` there holds for every call that returns normally, so it is the author saying what they diff --git a/pyproject.toml b/pyproject.toml index db5f15cdf6..9a5fa921d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,11 @@ release = { requires-python = ">=3.12" } [tool.ruff] target-version = "py38" extend-exclude = [ + # benchmark programs are measurement inputs, not code. an autofix that + # rewrites `i = i + 1` or hoists a loop changes what is being measured, and + # the isort rule alone would put `from __future__ import annotations` at the + # top of every one of them + "scripts/native-bench/programs/", "crates/ty_vendored/vendor/", "crates/ruff/resources/", "crates/ruff_linter/resources/", diff --git a/scripts/bg.sh b/scripts/bg.sh new file mode 100755 index 0000000000..eb58781312 --- /dev/null +++ b/scripts/bg.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# start a long job and wait for it across several turns, without burning the +# 600-second command bound on a wait that answers nothing +# +# a build, a test run or a sweep outlives the bound the tool harness puts on one +# command, so it has to run in the background and be polled. hand-rolled polling +# has been measured as **the single largest cost in this work**: across five +# agents and 12h27m of wall time, 7h13m went into `sleep`/`until` loops against +# 43m of `cargo build`. 3h of that was loops that ran to the bound and returned +# nothing, and 2h was loops that could never have returned at all: +# +# until ! pgrep -f "native-sweeps/isoimport.sh"; do sleep 30; done +# +# `pgrep -f` matches against whole command lines, and that loop's own command +# line contains the pattern — so it matches itself, and waits for the bound no +# matter what the sweep is doing. this file exists so nobody writes that again +# +# usage: +# scripts/bg.sh start launch, return immediately +# scripts/bg.sh wait [seconds] wait up to N (default 540), then +# report `running` and return 0 so +# the caller can simply call again +# scripts/bg.sh status done/running/missing, no waiting +# scripts/bg.sh log [lines] tail the output +# +# `wait` distinguishes the two outcomes a bare sleep loop cannot: `done rc=N` +# means the job really finished and N is its exit status; `running` means it is +# still going and nothing is wrong. a job that dies still writes its marker, so +# a crash is never mistaken for slowness + +set -u + +dir() { + # the scratchpad if the harness gave us one, else a temp dir. keeping the + # markers out of the project matters: `by compile` compiles every source it + # finds beside the file it was given + printf '%s\n' "${BY_BG_DIR:-${TMPDIR:-/tmp}/by-bg}" +} + +start() { + local name="$1"; shift + local d; d=$(dir); mkdir -p "$d" + rm -f "$d/$name.done" "$d/$name.log" "$d/$name.pid" + # the marker is written by the same shell that runs the command, after it + # exits, so it cannot be missed and it carries the real status + # + # the single quotes are the point: `$@`, `$?` and `$0` are for the *inner* shell to + # expand once it has run the command, not for this one to expand now + # shellcheck disable=SC2016 + nohup bash -c '"$@" ; printf "%s\n" "$?" > "$0"' \ + "$d/$name.done" "$@" > "$d/$name.log" 2>&1 & + printf '%s\n' "$!" > "$d/$name.pid" + printf 'started %s (log %s)\n' "$name" "$d/$name.log" +} + +# a job killed outright — OOM, SIGKILL, the machine giving up — never reaches the +# line that writes its marker, and without this it would report `running` until +# the caller gave up. so liveness is checked too, and by *pid*: `kill -0` takes a +# number, which cannot match the command line of the shell asking the question. +# that is the whole difference from `pgrep -f`, which can and does +status() { + local name="$1" d; d=$(dir) + if [ -f "$d/$name.done" ]; then + printf 'done rc=%s\n' "$(cat "$d/$name.done")" + return 0 + fi + if [ ! -f "$d/$name.log" ]; then + printf 'missing\n' + return 0 + fi + local pid="" + [ -f "$d/$name.pid" ] && pid=$(cat "$d/$name.pid") + if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then + # the marker is written *before* the shell exits, so seeing the exit first + # only means the write has not landed yet. re-read once before calling it a + # death, or a job that merely finished quickly is reported as killed + sleep 1 + if [ -f "$d/$name.done" ]; then + printf 'done rc=%s\n' "$(cat "$d/$name.done")" + return 0 + fi + printf 'died (no exit status — killed, not finished)\n' + return 0 + fi + printf 'running\n' +} + +wait_for() { + local name="$1" bound="${2:-540}" d; d=$(dir) + local waited=0 state + while [ "$waited" -lt "$bound" ]; do + state=$(status "$name") + case "$state" in + running) ;; + *) printf '%s\n' "$state"; return 0 ;; + esac + sleep 5 + waited=$((waited + 5)) + done + # not a failure — the caller polls again. returning 0 keeps a bounded wait + # from reading as a broken command + printf 'running (%ss elapsed, call again)\n' "$waited" + return 0 +} + +case "${1:-}" in + start) shift; start "$@" ;; + wait) shift; wait_for "$@" ;; + status) shift; status "$@" ;; + log) shift; tail -n "${2:-40}" "$(dir)/$1.log" ;; + *) printf 'usage: %s {start|wait|status|log} [...]\n' "$0" >&2; exit 2 ;; +esac diff --git a/scripts/native-bench/README.md b/scripts/native-bench/README.md new file mode 100644 index 0000000000..5b6c50e597 --- /dev/null +++ b/scripts/native-bench/README.md @@ -0,0 +1,19 @@ +# native-bench + +```sh +cargo build --release --bin by +uv run --no-project --python 3.13 python scripts/native-bench/bench.py +uv run --no-project --python 3.13 python scripts/native-bench/bench.py --self-check +``` + +times each benchmark four ways — interpreted, `by compile`, `by compile` again, +and mypyc — and reports paired ratios with the noise floor those two identical +builds measured for themselves + +the method, the guardrails and the reason for every benchmark in the set are in +[the docs](../../docs/basedpython/development/compilation/benchmarks.md). read +that before changing anything here, and before quoting a number out of it + +`--python` takes anything `uv python find` accepts, including a free-threaded +build (`3.13t`, `3.14t`) — the version is recorded in the run's metadata and a +baseline from a different one warns before it compares diff --git a/scripts/native-bench/bench.py b/scripts/native-bench/bench.py new file mode 100644 index 0000000000..b9404376e4 --- /dev/null +++ b/scripts/native-bench/bench.py @@ -0,0 +1,1106 @@ +"""the native benchmark suite: stage, build, prove, time, compare + + scripts/native-bench/bench.py # everything + scripts/native-bench/bench.py mandel dot # only these + scripts/native-bench/bench.py --json today.json + scripts/native-bench/bench.py --baseline last-week.json + +the method it enforces is written up in +`docs/basedpython/development/compilation/benchmarks.md`. the short version: + +- **the ratio is the measurement.** absolute times on this suite move by a + factor of four with machine load, so nothing is reported from one build's + clock alone. every ratio is *paired* — the four builds of a benchmark are + timed in one process, round by round, so a spike lands on all four and mostly + cancels in the quotient +- **the table carries its own error bar.** every benchmark is built twice by the + same compiler, and the second build is timed alongside the first. two builds + of the same source are the same program, so the ratio between them is the + suite's noise floor for that benchmark, measured rather than assumed. a + difference smaller than the floor is not a difference +- **nothing is timed until it has proved what it is.** a compiled build has to + import as a real extension module, from a file inside this run's own root, + newer than this run's build, and it has to return the same answer as the + interpreted one. every one of those is a refusal rather than a warning +- **a decline is a failure, not a footnote.** `programs.toml` records how many + functions each benchmark is expected to leave interpreted, and a run where the + count moved either way fails. otherwise a benchmark can quietly stop measuring + compiled code and go on posting numbers +- **the harness cannot match nothing.** an unknown benchmark name, a program + with no manifest entry, a manifest entry with no program, an empty selection: + all of them exit non-zero. a measurement harness that cannot fail loudly is + worse than none +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import shutil +import statistics +import subprocess +import sys +import tempfile +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +PROGRAMS = HERE / "programs" +MANIFEST = HERE / "programs.toml" + +# one sample is as many calls as it takes to land between these, measured +# against the fastest and the slowest build respectively. every build of a +# benchmark runs the same number, so the pairing stays exact +MIN_SAMPLE = 0.010 +MAX_SAMPLE = 0.150 +PROBE_SAMPLE = 0.010 +MAX_CALLS = 1_000_000 + +# below this the median's confidence interval degenerates to the full range of +# what was seen, which is not a confidence interval. a table built from three +# rounds is how this suite published 0.2x for something that was 0.9x +MIN_ROUNDS = 9 + +# the control bounds the noise *within* a run: same process, same moment, same +# memory layout. it does not bound the drift *between* two runs, which is larger +# and which is what a baseline comparison is actually up against — two runs of +# an unchanged compiler, forty minutes apart on a 16-core laptop, disagreed by +# 8.4% on `words` while each reported a ±0.8% floor for itself. so the default +# bar for calling a change a change is set from that measurement rather than +# from the within-run floor. tighten it on a machine that does nothing else +BETWEEN_RUN_DRIFT = 0.10 + + +class Failure(Exception): + """something the run cannot honestly continue past""" + + +def load() -> str: + """what else the machine is doing, recorded at both ends of the run + + it does not change any decision the harness makes — the noise floor does + that, and it measures the effect rather than guessing at it. this is here so + that a table someone kept can be read back with the reason it was wide + """ + try: + return f"{os.getloadavg()[0]:.2f}" + except (OSError, AttributeError): + return "unknown" + + +# ── statistics ─────────────────────────────────────────────────────────────── +# +# the median throughout, never the mean and never the minimum. the mean is moved +# by the one round that hit a scheduler; the minimum is an extreme-value +# statistic whose expectation depends on how many rounds were run and on how +# quiet the machine happened to be, so it is not comparable between runs — which +# is the only thing this suite is for. + + +def median_interval(values: list[float], alpha: float = 0.05) -> tuple[float, float]: + """a distribution-free confidence interval for the median + + the order statistics either side of the middle, chosen from the binomial + tail. exact, assumes nothing about the shape of the noise, needs no + resampling and no random numbers — so two runs over the same data agree + """ + ordered = sorted(values) + n = len(ordered) + if n < 6: + return ordered[0], ordered[-1] + cumulative, k = 0.0, 0 + for i in range(n + 1): + p = math.comb(n, i) / 2**n + if cumulative + p > alpha / 2: + break + cumulative += p + k = i + 1 + if k == 0: + return ordered[0], ordered[-1] + return ordered[k - 1], ordered[n - k] + + +@dataclass +class Ratio: + """a paired ratio and how sure of it the run is""" + + median: float + low: float + high: float + + @property + def spread(self) -> float: + """half the interval, as a fraction of the median""" + return (self.high - self.low) / 2 / self.median if self.median else 0.0 + + def render(self, places: int = 2) -> str: + return f"{self.median:.{places}f}x ±{self.spread * 100:.1f}%" + + def as_json(self) -> dict: + return {"median": self.median, "low": self.low, "high": self.high} + + +def ratio_json(result: Result, leg: str) -> dict | None: + """a leg's paired ratio as json, or `None` where there was no pairing + + the three json rows all read a ratio that may not exist — a leg the run did not + time has none — and two of the three used to read it without asking. one place + to forget rather than three + """ + ratio = result.ratio(leg) + return ratio.as_json() if ratio else None + + +def paired(numerator: list[float], denominator: list[float]) -> Ratio: + """the ratio of two builds, round by round rather than time against time + + dividing one build's median by another's would let a load spike that only + landed on one of them through unchallenged. these two lists are the same + rounds, so the spike is in both quotients and cancels + """ + ratios = [a / b for a, b in zip(numerator, denominator, strict=True)] + low, high = median_interval(ratios) + return Ratio(statistics.median(ratios), low, high) + + +# ── the manifest ───────────────────────────────────────────────────────────── + + +@dataclass +class Program: + name: str + group: str + measures: str + declines: int + mypyc: bool + + +def load_manifest(selected: list[str]) -> list[Program]: + """read `programs.toml`, and refuse to run against a corpus that has drifted + + the two lists — the files on disk and the entries in the manifest — are + meant to agree exactly. a file with no entry has no declared decline count + and so is unguarded; an entry with no file quietly measures nothing + """ + entries = tomllib.loads(MANIFEST.read_text())["programs"] + on_disk = {path.stem for path in PROGRAMS.glob("*.py")} + declared = set(entries) + + if undeclared := on_disk - declared: + raise Failure( + f"programs with no manifest entry: {', '.join(sorted(undeclared))}" + ) + if missing := declared - on_disk: + raise Failure(f"manifest entries with no program: {', '.join(sorted(missing))}") + + if selected: + if unknown := set(selected) - declared: + raise Failure(f"no such benchmark: {', '.join(sorted(unknown))}") + wanted = [name for name in entries if name in set(selected)] + else: + wanted = list(entries) + + if not wanted: + raise Failure("nothing selected") + + return [ + Program( + name=name, + group=entries[name]["group"], + measures=entries[name]["measures"], + declines=entries[name]["declines"], + mypyc=entries[name].get("mypyc", True), + ) + for name in wanted + ] + + +# ── staging and building ───────────────────────────────────────────────────── + + +@dataclass +class Leg: + """one build of one benchmark, and everything known about it""" + + name: str + module: str + compiled: bool + directory: Path + built: bool = False + error: str | None = None + log: Path | None = None + declines: list[str] = field(default_factory=list) + + def spec(self) -> dict: + return { + "name": self.name, + "module": self.module, + "compiled": self.compiled, + "dir": str(self.directory), + } + + +def stage(root: Path, program: Program, leg: str, python_version: str) -> Path: + """lay one build out as a project of its own, under a name of its own + + the name is what makes the whole method possible: four builds of the same + benchmark have to coexist in one process to be timed against each other, and + an extension module's init hook is found by its name, so they cannot share + one + """ + directory = root / program.name / leg + (directory / "dist").mkdir(parents=True) + module = f"{program.name}_{leg}" + shutil.copy(PROGRAMS / f"{program.name}.py", directory / f"{module}.py") + (directory / "pyproject.toml").write_text( + f'[project]\nname = "bench"\nversion = "0"\n' + f'requires-python = ">={python_version}"\n\n' + # a `float` annotation admits an `int` under python's numeric promotion, + # so without this every float place holds `int | float` and cannot be + # unboxed. mypyc reads a `float` annotation as a machine double, so this + # is what makes the two comparable at all — and it is recorded in the + # run's metadata because it changes the numbers + f"[tool.ty.analysis]\nstrict-float = true\n\n" + f'[tool.ty.environment]\npython-version = "{python_version}"\n' + ) + return directory + + +def build_by( + by: Path, directory: Path, module: str, python: str +) -> tuple[bool, str | None, list[str]]: + """compile one build, and read back what it refused to compile + + the decline list comes from the `--annotate` report rather than from the + diagnostics: the diagnostic renderer wraps and truncates, and a count taken + from it has been wrong before + """ + log = directory / "build.log" + result = subprocess.run( + [str(by), "compile", f"{module}.py", "-o", "out", "--annotate"], + cwd=directory, + env={**os.environ, "PYTHON": python}, + capture_output=True, + text=True, + ) + log.write_text(result.stdout + result.stderr) + if result.returncode != 0: + return False, f"`by compile` exited {result.returncode}", [] + + declines = [] + for report in (directory / "out").glob("*.annotated"): + section = report.read_text().partition("## left to the interpreted definition")[ + 2 + ] + for line in section.partition("\n##")[0].splitlines(): + if line.startswith("- "): + declines.append(line[2:]) + + artefacts = [ + p + for p in (directory / "out").iterdir() + if p.suffix in {".so", ".pyd", ".dylib"} + ] + if not artefacts: + return False, "`by compile` succeeded but left no extension module", declines + for artefact in artefacts: + shutil.copy(artefact, directory / "dist" / artefact.name) + return True, None, declines + + +def build_mypyc(directory: Path, module: str, python: str) -> tuple[bool, str | None]: + """compile one build with mypyc, and say so out loud when it will not + + the previous harness sent this to /dev/null, and a run in which mypyc was + simply broken was read as mypyc being unable to compile the program + """ + log = directory / "build.log" + result = subprocess.run( + [ + "uv", + "run", + "--no-project", + "--with", + "mypy", + "--with", + "setuptools", + "--python", + python, + "mypyc", + f"{module}.py", + ], + cwd=directory, + capture_output=True, + text=True, + ) + log.write_text(result.stdout + result.stderr) + artefacts = [ + p for p in directory.iterdir() if p.suffix in {".so", ".pyd", ".dylib"} + ] + if not artefacts: + return ( + False, + f"mypyc left no extension module (exit {result.returncode}); log at {log}", + ) + for artefact in artefacts: + shutil.copy(artefact, directory / "dist" / artefact.name) + return True, None + + +# ── driving the timer ──────────────────────────────────────────────────────── + + +def drive(python: str, spec: dict, work: Path, tag: str) -> dict: + path = work / f"spec-{tag}.json" + path.write_text(json.dumps(spec)) + result = subprocess.run( + [python, str(HERE / "timer.py"), str(path)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise Failure( + f"the timer failed for {tag} (exit {result.returncode}):\n{result.stderr}" + ) + return json.loads(result.stdout) + + +# ── the run ────────────────────────────────────────────────────────────────── + + +@dataclass +class Result: + program: Program + status: str + legs: dict[str, Leg] + times: dict[str, list[float]] = field(default_factory=dict) + calls: int = 0 + notes: list[str] = field(default_factory=list) + + def median(self, leg: str) -> float | None: + values = self.times.get(leg) + return statistics.median(values) if values else None + + def ratio(self, leg: str) -> Ratio | None: + if leg in self.times and "by" in self.times: + return paired(self.times[leg], self.times["by"]) + return None + + @property + def noise(self) -> float | None: + """what the suite reported for two builds that are the same program + + both how far the control landed from 1.00 and how far it wandered: a + build that reads consistently 2% away from its own twin is as much a + problem as one that reads all over the place + """ + control = self.ratio("control") + if control is None: + return None + return max(abs(control.median - 1), control.spread) + + def noisy(self, limit: float) -> bool: + return self.noise is not None and self.noise > limit + + +def run_program( + args, + program: Program, + root: Path, + python: str, + python_version: str, + built_after: float, +) -> Result: + legs: dict[str, Leg] = {} + + directory = stage(root, program, "cpython", python_version) + shutil.copy( + directory / f"{program.name}_cpython.py", + directory / "dist" / f"{program.name}_cpython.py", + ) + legs["cpython"] = Leg( + "cpython", f"{program.name}_cpython", False, directory / "dist", built=True + ) + + # `control` is a second, independent compile of the same source. it is the + # null: two builds of one program are one program, so whatever ratio the + # suite reports between them is noise it invented. that number is printed on + # every row, and nothing smaller than it is a finding + for leg in ("by", "control"): + directory = stage(root, program, leg, python_version) + module = f"{program.name}_{leg}" + built, error, declines = build_by(args.by, directory, module, python) + legs[leg] = Leg( + leg, + module, + True, + directory / "dist", + built, + error, + directory / "build.log", + declines, + ) + + if program.mypyc and not args.no_mypyc: + directory = stage(root, program, "mypyc", python_version) + module = f"{program.name}_mypyc" + built, error = build_mypyc(directory, module, python) + legs["mypyc"] = Leg( + "mypyc", + module, + True, + directory / "dist", + built, + error, + directory / "build.log", + ) + + result = Result(program, "ok", legs) + + # a decline that nobody declared means the row below is not measuring what + # its name says. exact match in both directions: a compiler that started + # compiling something is as much a change as one that stopped + found = len(legs["by"].declines) + if legs["by"].built and found != program.declines: + result.status = "declines" + result.notes.append( + f"expected {program.declines} declined function(s), found {found}" + + (": " + "; ".join(legs["by"].declines) if legs["by"].declines else "") + ) + return result + + for leg in legs.values(): + if not leg.built: + if leg.name in ("by", "control"): + result.status = "build" + result.notes.append(f"{leg.name}: {leg.error}") + return result + result.notes.append(f"{leg.name}: {leg.error}") + + live = [leg for leg in legs.values() if leg.built] + base = { + "root": str(root), + "built_after": built_after, + "legs": [leg.spec() for leg in live], + } + + probe = drive(python, {**base, "mode": "probe"}, root, f"{program.name}-probe") + for name, why in probe["refused"].items(): + if name in ("by", "control", "cpython"): + result.status = "refused" + result.notes.append(f"{name}: {why}") + return result + result.notes.append(f"{name}: {why}") + live = [leg for leg in live if leg.name != name] + + # a build that got a different answer is not a faster build. `bigint` leans + # on this: a compiler that wrapped at 64 bits fails here rather than posting + # a very good time + expected = probe["answers"]["cpython"] + for name, answer in probe["answers"].items(): + if answer != expected: + result.status = "disagrees" + result.notes.append( + f"{name} answered {answer}, cpython answered {expected}" + ) + return result + + if args.verify_only: + return result + + base["legs"] = [leg.spec() for leg in live] + calibration = drive( + python, + { + **base, + "mode": "calibrate", + "probe_target": PROBE_SAMPLE, + "min_sample": MIN_SAMPLE, + "max_sample": MAX_SAMPLE, + "max_count": MAX_CALLS, + }, + root, + f"{program.name}-calibrate", + ) + result.calls = calibration["count"] + + timed = drive( + python, + { + **base, + "mode": "time", + "count": result.calls, + "rounds": args.rounds, + "warmup": args.warmup, + }, + root, + f"{program.name}-time", + ) + result.times = timed["timings"] + return result + + +# ── reporting ──────────────────────────────────────────────────────────────── + + +def render(results: list[Result], metadata: dict, show_declines: bool, limit: float): + header = ( + f"{'benchmark':<15}{'group':<10}{'cpython':>10}{'by':>10}{'mypyc':>10}" + f" {'vs cpython':>15}{'vs mypyc':>16}{'noise':>10}{'dec':>5}" + ) + print() + print(f"python {metadata['python']} ({metadata['implementation']})") + print(f"by {metadata['by_version']} from {metadata['git']}") + print(f"mypy {metadata['mypy'] or 'unavailable'}") + print( + f"host {metadata['host']}, {metadata['cpus']} cpus, load {metadata['load_before']}" + ) + print( + f"method {metadata['rounds']} rounds after {metadata['warmup']} warmup, " + f"paired medians, strict-float on" + ) + print() + print(header) + print("-" * len(header)) + + for result in results: + name, group = result.program.name, result.program.group + if result.status != "ok": + print( + f"{name:<15}{group:<10} {result.status.upper()}: {'; '.join(result.notes)}" + ) + continue + cpython, by = result.median("cpython"), result.median("by") + mypyc = result.median("mypyc") + against_cpython = result.ratio("cpython") + against_mypyc = result.ratio("mypyc") + noise = result.noise + # `ok` says every leg ran, not that every leg was *timed* — a leg whose samples + # were all discarded leaves no median behind. rather than assume the status + # covers it, say so in the row: a blank number is a measurement nobody has, + # which is not the same as a slow one + if cpython is None or by is None or noise is None: + print(f"{name:<15}{group:<10} no timing recorded") + continue + print( + f"{name:<15}{group:<10}" + f"{cpython * 1000:>9.2f}m{by * 1000:>9.2f}m" + f"{(f'{mypyc * 1000:.2f}m' if mypyc else '-'):>10}" + f" {(against_cpython.render() if against_cpython else '-'):>15}" + f"{(against_mypyc.render() if against_mypyc else '-'):>16}" + f"{f'±{noise * 100:.1f}%' + ('!' if result.noisy(limit) else ''):>10}" + f"{len(result.legs['by'].declines):>5}" + ) + for note in result.notes: + print(f"{'':<25}note: {note}") + + print() + floors = [r.noise for r in results if r.status == "ok" and r.noise is not None] + if floors: + print( + f"noise floor: median ±{statistics.median(floors) * 100:.2f}%, " + f"worst ±{max(floors) * 100:.2f}% — a change smaller than a row's own " + f"floor is not a change" + ) + failed = [r for r in results if r.status != "ok"] + noisy = [r for r in results if r.status == "ok" and r.noisy(limit)] + print( + f"{len(results) - len(failed) - len(noisy)}/{len(results)} benchmarks measured" + ) + if noisy: + # the row's two identical builds disagreed by more than the limit, so + # whatever else it says, it is not a measurement of anything. marked + # rather than deleted: the numbers are still worth a glance, and the + # `!` is there to stop one being quoted + print( + f"{len(noisy)} too noisy to trust (marked `!`): " + f"{', '.join(r.program.name for r in noisy)} — rerun on a quieter machine" + ) + + if show_declines: + for result in results: + if result.legs.get("by") and result.legs["by"].declines: + print(f"\n{result.program.name} declines:") + for decline in result.legs["by"].declines: + print(f" {decline}") + + +def render_verification(results: list[Result]): + """everything the suite checks that does not involve a clock + + this half is deterministic, so it is the half that can run anywhere — a + shared runner cannot time anything, but it can tell you that a benchmark + stopped compiling, or that the compiled build started answering differently + """ + print() + for result in results: + declines = len(result.legs["by"].declines) if "by" in result.legs else 0 + mark = "ok " if result.status == "ok" else result.status.upper() + print( + f"{mark:<10}{result.program.name:<15}{declines:>3} declined" + + (f" {'; '.join(result.notes)}" if result.notes else "") + ) + failed = [r for r in results if r.status != "ok"] + print(f"\n{len(results) - len(failed)}/{len(results)} verified") + + +def compare( + results: list[Result], baseline: dict, threshold: float, limit: float +) -> bool: + """a regression is detected rather than eyeballed + + the quantity compared between runs is the speedup, not the time: absolute + times are not comparable across machines or across a Tuesday, and this suite + exists because they were compared anyway + """ + print("\nagainst the baseline") + for key in ("python", "implementation", "host"): + if baseline["metadata"].get(key) != CURRENT[key]: + print( + f" warning: {key} was {baseline['metadata'].get(key)!r} then and is " + f"{CURRENT[key]!r} now — the two runs are not comparable" + ) + + print(f" {'benchmark':<15}{'then':>12}{'now':>12}{'change':>12} verdict") + regressed = False + for result in results: + previous = baseline["benchmarks"].get(result.program.name) + if result.status != "ok" or previous is None or previous.get("status") != "ok": + continue + against_cpython = result.ratio("cpython") + new_noise = result.noise + # the same gap as in the table above: `ok` does not promise a timing survived. + # a row with nothing to compare is left out of the comparison rather than + # compared against a number that is not there + if against_cpython is None or new_noise is None: + continue + old = previous["vs_cpython"]["median"] + new = against_cpython.median + change = new / old - 1 + old_noise = previous["noise"] + # a row either run could not measure is skipped rather than compared + # with a very wide bar. a pair of identical builds that disagreed by 70% + # says the machine was preempting samples, and under preemption the + # longer sample loses more — so the paired ratio does not merely get + # noisier, it drifts upwards. widening the bar does not fix a bias + if max(old_noise, new_noise) > limit: + print( + f" {result.program.name:<15}{old:>11.2f}x{new:>11.2f}x" + f"{change * 100:>11.1f}% skipped: ±{max(old_noise, new_noise) * 100:.0f}% " + f"noise in one of the two runs" + ) + continue + # the bar is whichever is larger: the noise the two runs measured for + # themselves, or the floor asked for on the command line + bar = max(old_noise + new_noise, threshold) + if change < -bar: + verdict, regressed = "REGRESSED", True + elif change > bar: + verdict = "improved" + else: + verdict = f"same (within ±{bar * 100:.1f}%)" + print( + f" {result.program.name:<15}{old:>11.2f}x{new:>11.2f}x" + f"{change * 100:>11.1f}% {verdict}" + ) + return regressed + + +CURRENT: dict = {} + + +# ── the harness's check on itself ──────────────────────────────────────────── + + +def self_check(by: Path, python: str, python_version: str) -> int: + """build each way this suite has lied before, and prove it is refused now + + an ablation harness here once matched nothing and reported that every edit + cost nothing, and the reason it was believed is that a harness which cannot + fail looks exactly like a harness that found nothing wrong. so the refusals + are exercised rather than asserted, by handing the timer a leg that is + wrong in each of the ways a leg has actually been wrong + """ + root = Path(tempfile.mkdtemp(prefix="native-bench-selfcheck-")) + failures = [] + + def expect(what: str, refusal: str | None, wanted: str): + if refusal is None: + failures.append(f"{what}: was accepted, and should have been refused") + elif wanted not in refusal: + failures.append(f"{what}: refused with {refusal!r}, wanted {wanted!r}") + else: + print(f" refused {what}: {refusal}") + + def refusal_for(leg: dict, built_after: float = 0.0) -> str | None: + answer = drive( + python, + { + "mode": "probe", + "root": str(root), + "built_after": built_after, + "legs": [leg], + }, + root, + "selfcheck", + ) + return answer["refused"].get(leg["name"]) + + # a real extension module to be wrong about + stage = root / "real" + (stage / "dist").mkdir(parents=True) + (stage / "canary.py").write_text("def bench() -> int:\n return 1\n") + (stage / "pyproject.toml").write_text( + f'[project]\nname = "c"\nversion = "0"\nrequires-python = ">={python_version}"\n' + ) + built, error, _ = build_by(by, stage, "canary", python) + if not built: + print( + f"error: the self-check could not build its own canary: {error}", + file=sys.stderr, + ) + return 2 + + print("the guards:") + expect( + "an interpreted build passed off as a compiled one", + refusal_for( + {"name": "by", "module": "canary", "compiled": True, "dir": str(stage)} + ), + "not an extension module", + ) + expect( + "a compiled build passed off as an interpreted one", + refusal_for( + { + "name": "cpython", + "module": "canary", + "compiled": False, + "dir": str(stage / "dist"), + } + ), + "is an extension module", + ) + # the stale-output-directory failure: the build did not happen, and the + # artefact left over from last time answered in its place + expect( + "an artefact older than this run's build", + refusal_for( + { + "name": "by", + "module": "canary", + "compiled": True, + "dir": str(stage / "dist"), + }, + built_after=os.stat(stage).st_mtime + 3600, + ), + "predates this run's build", + ) + outside = Path(tempfile.mkdtemp(prefix="native-bench-outside-")) + for artefact in (stage / "dist").glob("canary*"): + shutil.copy(artefact, outside / artefact.name) + expect( + "an artefact from outside this run's root", + refusal_for( + {"name": "by", "module": "canary", "compiled": True, "dir": str(outside)} + ), + "outside this run's root", + ) + expect( + "a build that is not there at all", + refusal_for( + { + "name": "by", + "module": "absent", + "compiled": True, + "dir": str(stage / "dist"), + } + ), + "import failed", + ) + + # and the corpus guards, which are what stops a run measuring nothing + for what, selection, wanted in ( + ("an unknown benchmark name", ["nosuchbenchmark"], "no such benchmark"), + ): + try: + load_manifest(selection) + failures.append(f"{what}: was accepted") + except Failure as failure: + if wanted not in str(failure): + failures.append(f"{what}: refused with {failure!r}") + else: + print(f" refused {what}: {failure}") + + shutil.rmtree(root, ignore_errors=True) + shutil.rmtree(outside, ignore_errors=True) + for failure in failures: + print(f"error: {failure}", file=sys.stderr) + print( + "\nevery guard held" + if not failures + else f"\n{len(failures)} guard(s) did not hold" + ) + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "programs", nargs="*", help="benchmarks to run (default: all of them)" + ) + parser.add_argument( + "--by", + type=Path, + default=ROOT / "target" / "release" / "by", + help="the compiler to measure (default: this checkout's release build)", + ) + parser.add_argument( + "--python", + default="3.13", + help="interpreter to build against and run: a version for `uv python find`, or a path", + ) + parser.add_argument( + "--rounds", + type=int, + default=21, + help="timed rounds per benchmark (default: 21)", + ) + parser.add_argument( + "--warmup", type=int, default=3, help="untimed rounds first (default: 3)" + ) + parser.add_argument("--no-mypyc", action="store_true", help="skip the mypyc build") + parser.add_argument( + "--declines", action="store_true", help="list every declined function" + ) + parser.add_argument( + "--json", type=Path, help="write the full result for a later comparison" + ) + parser.add_argument( + "--baseline", type=Path, help="compare against a previous --json" + ) + parser.add_argument( + "--regression-threshold", + type=float, + default=BETWEEN_RUN_DRIFT, + help=f"the smallest change worth calling one, when noise is smaller " + f"(default: {BETWEEN_RUN_DRIFT})", + ) + parser.add_argument( + "--noise-limit", + type=float, + default=0.10, + help="the noise a row may show and still count as measured (default: 0.10)", + ) + parser.add_argument( + "--keep", action="store_true", help="keep the build tree and say where it is" + ) + parser.add_argument( + "--self-check", + action="store_true", + help="prove the guards refuse what they are meant to, and run nothing else", + ) + parser.add_argument( + "--verify-only", + action="store_true", + help="build, prove and check declines, but time nothing (deterministic, so it runs anywhere)", + ) + args = parser.parse_args() + + if args.rounds < MIN_ROUNDS: + raise Failure( + f"--rounds {args.rounds} is not a measurement: below {MIN_ROUNDS} rounds the " + "confidence interval degenerates to the range of what was seen" + ) + + # self-check returns before anything reads this, but an empty list rather than an + # unbound name says that here instead of leaving it to be inferred from control flow + programs: list[Program] = [] + if not args.self_check: + programs = load_manifest(args.programs) + + if not args.by.is_file() or not os.access(args.by, os.X_OK): + raise Failure( + f"no compiler at {args.by} — build one with `cargo build --release --bin by`.\n" + "a debug build is far too slow to measure anything with" + ) + + if Path(args.python).is_absolute(): + python = args.python + else: + found = subprocess.run( + ["uv", "python", "find", args.python], capture_output=True, text=True + ) + if found.returncode != 0: + raise Failure(f"no interpreter for {args.python!r}: {found.stderr.strip()}") + python = found.stdout.strip() + + probe = subprocess.run( + [ + python, + "-c", + "import platform,sys;" + "print(platform.python_version());print(platform.python_implementation());" + "print(sys.version.split()[0])", + ], + capture_output=True, + text=True, + ) + if probe.returncode != 0: + raise Failure(f"{python} does not run: {probe.stderr.strip()}") + # the probe below prints three fields, but a python that printed something else + # would unpack into an unhelpful ValueError here rather than say what it answered + fields = probe.stdout.split() + if len(fields) < 2: + raise Failure( + f"{python} answered {probe.stdout.strip()!r}, " + "not a version and an implementation" + ) + full_version, implementation = fields[0], fields[1] + python_version = ".".join(full_version.split(".")[:2]) + + if args.self_check: + return self_check(args.by, python, python_version) + + version = subprocess.run( + [str(args.by), "--version"], capture_output=True, text=True + ).stdout.strip() + git = subprocess.run( + ["git", "-C", str(ROOT), "describe", "--always", "--dirty"], + capture_output=True, + text=True, + ).stdout.strip() + mypy = "skipped" if args.no_mypyc else None + if not args.no_mypyc: + found = subprocess.run( + [ + "uv", + "run", + "--no-project", + "--with", + "mypy", + "--python", + python, + "mypy", + "--version", + ], + capture_output=True, + text=True, + ) + mypy = found.stdout.strip() if found.returncode == 0 else None + if mypy is None: + print( + f"warning: mypyc is unavailable, so nothing will be compared against it\n{found.stderr}", + file=sys.stderr, + ) + + CURRENT.update( + { + "python": full_version, + "implementation": implementation, + "by_version": version, + "git": git or "unknown", + "mypy": mypy, + "host": f"{platform.system()} {platform.machine()}", + "cpus": os.cpu_count(), + "load_before": load(), + "rounds": args.rounds, + "warmup": args.warmup, + "strict_float": True, + } + ) + + # a fresh root every run, so a stale artefact cannot be picked up by + # construction rather than by remembering to delete one + root = Path(tempfile.mkdtemp(prefix="native-bench-")) + built_after = root.stat().st_mtime + + results = [] + for index, program in enumerate(programs, 1): + print(f"[{index}/{len(programs)}] {program.name}", file=sys.stderr) + results.append( + run_program(args, program, root, python, python_version, built_after) + ) + + CURRENT["load_after"] = load() + CURRENT["noise_limit"] = args.noise_limit + if args.verify_only: + render_verification(results) + if args.keep: + print(f"build tree kept at {root}") + else: + shutil.rmtree(root, ignore_errors=True) + failed = [r for r in results if r.status != "ok"] + return 1 if failed else 0 + render(results, CURRENT, args.declines, args.noise_limit) + + payload = { + "metadata": CURRENT, + "benchmarks": { + result.program.name: { + "status": result.status, + "group": result.program.group, + "notes": result.notes, + "calls": result.calls, + "declines": result.legs["by"].declines if "by" in result.legs else [], + "times": result.times, + **( + { + "vs_cpython": ratio_json(result, "cpython"), + "vs_mypyc": ratio_json(result, "mypyc"), + "control": ratio_json(result, "control"), + "noise": result.noise, + "noisy": result.noisy(args.noise_limit), + } + if result.status == "ok" + else {} + ), + } + for result in results + }, + } + if args.json: + args.json.write_text(json.dumps(payload, indent=2)) + print(f"written to {args.json}") + + regressed = False + if args.baseline: + regressed = compare( + results, + json.loads(args.baseline.read_text()), + args.regression_threshold, + args.noise_limit, + ) + + if args.keep: + print(f"build tree kept at {root}") + else: + shutil.rmtree(root, ignore_errors=True) + + failed = [r for r in results if r.status != "ok"] + noisy = [r for r in results if r.status == "ok" and r.noisy(args.noise_limit)] + if failed: + print( + f"\n{len(failed)} benchmark(s) did not measure: " + f"{', '.join(r.program.name for r in failed)}", + file=sys.stderr, + ) + # a run with a junk row in it exits non-zero even when every row it *could* + # measure looks fine. the alternative is a table that is mostly trustworthy, + # which is the kind nobody remembers to check before quoting + return 1 if failed or noisy or regressed else 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as failure: + print(f"error: {failure}", file=sys.stderr) + raise SystemExit(2) from None diff --git a/scripts/native-bench/programs.toml b/scripts/native-bench/programs.toml new file mode 100644 index 0000000000..580657f3fa --- /dev/null +++ b/scripts/native-bench/programs.toml @@ -0,0 +1,150 @@ +# the benchmark corpus, and the ledger of what the native backend refuses +# +# `group` is the axis a benchmark exists to measure. `measures` is the one-line +# reason it is in the set — if two entries would have the same one, one of them +# should go. +# +# `declines` is how many functions `by compile` currently leaves to their +# interpreted definitions. it is checked exactly: a benchmark whose count moved +# in either direction fails the run rather than quietly reporting a number about +# interpreted code. improve the compiler and this file changes in the same +# commit, which makes it a readable history of what the backend learned to take. +# +# `mypyc = false` means mypyc genuinely cannot build the program — the harness +# reports a *failed* mypyc build loudly, so this is for the cases that are not +# failures, and nothing may be listed here without the log to justify it. + +[programs.mandel] +group = "float" +measures = "scalar float arithmetic with a call in the inner loop" +declines = 0 + +[programs.mandel_inline] +group = "float" +measures = "the same work with the loop spelled out — the pair isolates call cost" +declines = 0 + +[programs.loops] +group = "int" +measures = "tagged-integer arithmetic with no container, no call and no float" +declines = 0 + +[programs.bigint] +group = "int" +measures = "integer arithmetic that leaves the machine word" +declines = 0 + +[programs.calls] +group = "dispatch" +measures = "a plain function call with a one-add body" +declines = 0 + +[programs.methods] +group = "dispatch" +measures = "a method call on a long-lived object" +declines = 0 + +[programs.recurse] +group = "dispatch" +measures = "calls that nest rather than repeat" +declines = 0 + +[programs.inherit] +group = "dispatch" +measures = "an override reached through a base-typed local" +declines = 0 + +[programs.alloc] +group = "memory" +measures = "an object built and dropped every iteration" +declines = 0 + +[programs.fields] +group = "memory" +measures = "attribute reads and writes on one long-lived object" +declines = 0 + +[programs.objects] +group = "memory" +measures = "allocation, fields and methods together, as object code looks" +declines = 0 + +[programs.dot] +group = "list" +measures = "indexed reads over two lists that never escape" +declines = 0 + +[programs.prefix] +group = "list" +measures = "appends into a list the function owns, iterated with `for`" +declines = 0 + +[programs.sieve] +group = "list" +measures = "indexed writes into a `bool` buffer from a nested loop" +declines = 0 + +[programs.tuples] +group = "tuple" +measures = "tuples packed, unpacked and indexed, and a two-value return" +declines = 0 + +[programs.sets] +group = "set" +measures = "a set built once and tested for membership many times" +declines = 0 + +[programs.dictget] +group = "dict" +measures = "dict lookup and in-place update, keys handed in already built" +declines = 0 + +[programs.dicthist] +group = "dict" +measures = "the histogram shape: membership test then read-modify-write" +declines = 0 + +[programs.words] +group = "str" +measures = "string concatenation in a loop" +declines = 0 + +[programs.chars] +group = "str" +measures = "an indexed scan over text, a character at a time" +declines = 0 + +[programs.strops] +group = "str" +measures = "the `str` methods real code calls: split, join, startswith, upper" +declines = 0 + +[programs.keybuild] +group = "str" +measures = "a builtin through the module namespace, a call, and a concatenation" +declines = 0 + +[programs.generic] +group = "boxing" +measures = "a list handed to a function with a type parameter" +declines = 0 + +[programs.generic_mono] +group = "boxing" +measures = "the same call monomorphised — the pair isolates what boxing costs" +declines = 0 + +[programs.excs] +group = "control" +measures = "a raise that is caught, and a `try` that never fires" +declines = 0 + +[programs.gen] +group = "frames" +measures = "a generator consumed by a `for` loop" +declines = 0 + +[programs.coro] +group = "frames" +measures = "awaits that complete immediately, driven without an event loop" +declines = 0 diff --git a/scripts/native-bench/programs/alloc.py b/scripts/native-bench/programs/alloc.py new file mode 100644 index 0000000000..dedc6111b3 --- /dev/null +++ b/scripts/native-bench/programs/alloc.py @@ -0,0 +1,27 @@ +"""an object built and dropped every iteration, and nothing kept + +allocation, the field stores the constructor makes, and the deallocation that +follows when the last reference goes. `objects` allocates too but then calls +methods on what it built, so its number is a mixture; this one is the mixture's +allocation half on its own +""" + + +class Pair: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + pair = Pair(i, i + 1) + total = total + pair.x + i = i + 1 + return total + + +def bench() -> int: + return run(300000) diff --git a/scripts/native-bench/programs/bigint.py b/scripts/native-bench/programs/bigint.py new file mode 100644 index 0000000000..6e85247fb1 --- /dev/null +++ b/scripts/native-bench/programs/bigint.py @@ -0,0 +1,33 @@ +"""integer arithmetic that leaves the machine word + +a compiler that unboxes `int` has to decide what happens when the value stops +fitting. the answer is checked as well as timed: `bench()` returns a number far +wider than 64 bits, so a build that wrapped instead of promoting fails the +agreement check rather than posting a fast time +""" + + +def factorial(n: int) -> int: + out = 1 + i = 2 + while i <= n: + out = out * i + i = i + 1 + return out + + +def digits(value: int) -> int: + count = 0 + while value > 0: + value = value // 10 + count = count + 1 + return count + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + total = total + digits(factorial(300)) + r = r + 1 + return total + factorial(60) diff --git a/scripts/native-bench/programs/calls.py b/scripts/native-bench/programs/calls.py new file mode 100644 index 0000000000..ab749e196a --- /dev/null +++ b/scripts/native-bench/programs/calls.py @@ -0,0 +1,24 @@ +"""a plain function call, with as little else in the loop as it can have + +the body is one add, so what the loop measures is the calling convention: the +frame, the argument passing, and the error check after the return. `mandel` has +a call on its hot path too but pays for float arithmetic around it, which is the +thing that hid a call regression once already +""" + + +def add(a: int, b: int) -> int: + return a + b + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = add(total, i) + i = i + 1 + return total + + +def bench() -> int: + return run(400000) diff --git a/scripts/native-bench/programs/chars.py b/scripts/native-bench/programs/chars.py new file mode 100644 index 0000000000..87a5ab9600 --- /dev/null +++ b/scripts/native-bench/programs/chars.py @@ -0,0 +1,37 @@ +"""a scan over text, a character at a time: indexing and comparison + +the text is made by repetition rather than by concatenation so that `words` — +which is the concatenation benchmark — cannot leak into this measurement +""" + + +def text(n: int) -> str: + unit = "word0 word1 word2 word3 word4 word5 word6 word7 word8 word9 " + return unit * n + + +def longest_run(line: str) -> int: + best = 0 + run = 0 + i = 0 + while i < len(line): + if line[i] == " ": + if run > best: + best = run + run = 0 + else: + run = run + 1 + i = i + 1 + if run > best: + best = run + return best + + +def bench() -> int: + line = text(2000) + total = 0 + r = 0 + while r < 10: + total = total + longest_run(line) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/coro.py b/scripts/native-bench/programs/coro.py new file mode 100644 index 0000000000..8b6dc730fe --- /dev/null +++ b/scripts/native-bench/programs/coro.py @@ -0,0 +1,40 @@ +"""awaits that complete immediately, driven without an event loop + +the loop is driven by hand rather than through `asyncio` on purpose: an event +loop would swamp the thing being measured with scheduling, and most awaits in +real code complete without ever suspending. what this times is the frame — its +creation, its resume, and the await of a coroutine that is already done +""" + + +async def step(i: int) -> int: + return (i * 7) % 13 + + +async def chain(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await step(i) + i = i + 1 + return total + + +def drive(n: int) -> int: + """run a coroutine to completion, with no loop underneath it""" + coroutine = chain(n) + try: + coroutine.send(None) + except StopIteration as done: + return done.value + coroutine.close() + return -1 + + +def bench() -> int: + total = 0 + run = 0 + while run < 20: + total = total + drive(2000) + run = run + 1 + return total diff --git a/scripts/native-bench/programs/dictget.py b/scripts/native-bench/programs/dictget.py new file mode 100644 index 0000000000..97c8e0b109 --- /dev/null +++ b/scripts/native-bench/programs/dictget.py @@ -0,0 +1,33 @@ +"""dict lookup and in-place update, with the keys handed in already built + +the keys are built once and the timed loop only subscripts, so this is the +lookup question and nothing else. `str` keys, because that is what a real table +is keyed by and where cpython's own lookup is most specialised +""" + + +def total(table: dict[str, int], keys: list[str], passes: int) -> int: + running = 0 + p = 0 + n = len(keys) + while p < passes: + i = 0 + while i < n: + key = keys[i] + running = running + table[key] + table[key] = table[key] + 1 + i = i + 1 + p = p + 1 + return running + + +def bench() -> int: + keys = [] + table: dict[str, int] = {} + i = 0 + while i < 2000: + key = "k" + str(i) + keys.append(key) + table[key] = i + i = i + 1 + return total(table, keys, 50) diff --git a/scripts/native-bench/programs/dicthist.py b/scripts/native-bench/programs/dicthist.py new file mode 100644 index 0000000000..1c101f01d1 --- /dev/null +++ b/scripts/native-bench/programs/dicthist.py @@ -0,0 +1,34 @@ +"""the histogram shape: a membership test, then a read-modify-write on the hit + +the word list is built once and counted many times, so counting is what the +number is about. the miss on the first sighting of each word is the branch that +makes this shape its own, and is why it is not folded into `dictget` +""" + + +def counted(words: list[str]) -> int: + seen: dict[str, int] = {} + for word in words: + if word in seen: + seen[word] = seen[word] + 1 + else: + seen[word] = 1 + return len(seen) + + +def repeated(words: list[str], passes: int) -> int: + total = 0 + p = 0 + while p < passes: + total = total + counted(words) + p = p + 1 + return total + + +def bench() -> int: + words = [] + i = 0 + while i < 20000: + words.append("w" + str(i % 500)) + i = i + 1 + return repeated(words, 10) diff --git a/scripts/native-bench/programs/dot.py b/scripts/native-bench/programs/dot.py new file mode 100644 index 0000000000..2701183cf8 --- /dev/null +++ b/scripts/native-bench/programs/dot.py @@ -0,0 +1,31 @@ +"""indexed reads over two lists that never escape the function + +the list is the one container the compiler can hold as a buffer, so this is the +best case for it: no growth, no writes, and the element type known at the +definition +""" + + +def dot(a: list[float], b: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(a): + out = out + a[i] * b[i] + i = i + 1 + return out + + +def bench() -> float: + a = [] + b = [] + i = 0 + while i < 50000: + a.append(i * 0.5) + b.append(i * 0.25) + i = i + 1 + total = 0.0 + r = 0 + while r < 10: + total = total + dot(a, b) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/excs.py b/scripts/native-bench/programs/excs.py new file mode 100644 index 0000000000..7c38d05181 --- /dev/null +++ b/scripts/native-bench/programs/excs.py @@ -0,0 +1,55 @@ +"""a raise that is caught, on the path the program expects to take + +every other benchmark stays on the straight line. this one leaves it: the +compiler emits an error edge from any fallible operation, and a `try` turns that +edge into a branch to a handler rather than to the function's own exit. + +two shapes, because they cost differently. `caught` raises and catches across a +call boundary, which is the expensive one. `guarded` never raises at all and +measures what merely being inside a `try` costs a loop that succeeds — which +should be nothing, and is worth knowing rather than assuming +""" + + +class Refused(Exception): + pass + + +def parse(text: str) -> int: + if len(text) == 0: + raise Refused("empty") + total = 0 + i = 0 + while i < len(text): + total = total + ord(text[i]) + i = i + 1 + return total + + +def caught(rounds: int) -> int: + total = 0 + i = 0 + while i < rounds: + try: + total = total + parse("" if i % 4 == 0 else "abc") + except Refused: + total = total + 1 + i = i + 1 + return total + + +def guarded(rounds: int) -> int: + """the same loop with a handler that is never reached""" + total = 0 + i = 0 + while i < rounds: + try: + total = total + parse("abc") + except Refused: + total = total + 1 + i = i + 1 + return total + + +def bench() -> int: + return caught(20000) + guarded(20000) diff --git a/scripts/native-bench/programs/fields.py b/scripts/native-bench/programs/fields.py new file mode 100644 index 0000000000..d908af06a3 --- /dev/null +++ b/scripts/native-bench/programs/fields.py @@ -0,0 +1,27 @@ +"""attribute reads and writes on one object that outlives the loop + +nothing is allocated inside the timed loop and no method is called, so this is +the field access on its own: on an emitted class that is an offset into a fixed +layout, and on an interpreted one it is a dict lookup through `__dict__` +""" + + +class State: + def __init__(self): + self.a = 0 + self.b = 1 + self.c = 2 + + +def run(state: State, n: int) -> int: + i = 0 + while i < n: + state.a = state.b + state.c + state.b = state.a - state.c + state.c = state.c + 1 + i = i + 1 + return state.a + state.b + state.c + + +def bench() -> int: + return run(State(), 300000) diff --git a/scripts/native-bench/programs/gen.py b/scripts/native-bench/programs/gen.py new file mode 100644 index 0000000000..24ffd768f2 --- /dev/null +++ b/scripts/native-bench/programs/gen.py @@ -0,0 +1,30 @@ +"""a generator consumed by a `for` loop + +the compiler turns a resumable frame into a state object: locals parked in +fields, a tag saying whether a suspension was a `yield` or an `await`, and a +resume method. `coro` covers the await half of that; this is the yield half, and +it is also the half real code writes far more often +""" + + +def steps(n: int): + i = 0 + while i < n: + yield (i * 7) % 13 + i = i + 1 + + +def consume(n: int) -> int: + total = 0 + for value in steps(n): + total = total + value + return total + + +def bench() -> int: + total = 0 + r = 0 + while r < 20: + total = total + consume(2000) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/generic.py b/scripts/native-bench/programs/generic.py new file mode 100644 index 0000000000..ff007240ab --- /dev/null +++ b/scripts/native-bench/programs/generic.py @@ -0,0 +1,32 @@ +"""a list built in a hot loop and handed to a generic function + +the type parameter is what stops the list being a buffer: the element +representation is not known at the definition, so neither the callee's body nor +the caller's list can be unboxed. `generic_mono` is the same program with the +call monomorphised by hand, and the gap between the two is the whole measurement +""" + + +def consume[T](xs: list[T], ys: list[T]) -> int: + n = 0 + i = 0 + while i < len(xs): + n = n + 1 + i = i + 1 + return n + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + xs = [0.0] + ys = [0.0] + i = 0 + while i < 2000: + xs.append(i * 0.5) + ys.append(i * 0.25) + i = i + 1 + total = total + consume(xs, ys) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/generic_mono.py b/scripts/native-bench/programs/generic_mono.py new file mode 100644 index 0000000000..e2b1047875 --- /dev/null +++ b/scripts/native-bench/programs/generic_mono.py @@ -0,0 +1,31 @@ +"""`generic`, with the call monomorphised by hand + +identical in every other respect, on purpose: read the two rows as a pair and +the difference is what the type parameter costs. read either one alone and it +says almost nothing +""" + + +def consume(xs: list[float], ys: list[float]) -> int: + n = 0 + i = 0 + while i < len(xs): + n = n + 1 + i = i + 1 + return n + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + xs = [0.0] + ys = [0.0] + i = 0 + while i < 2000: + xs.append(i * 0.5) + ys.append(i * 0.25) + i = i + 1 + total = total + consume(xs, ys) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/inherit.py b/scripts/native-bench/programs/inherit.py new file mode 100644 index 0000000000..dca699b8e7 --- /dev/null +++ b/scripts/native-bench/programs/inherit.py @@ -0,0 +1,44 @@ +"""an override reached through a base-typed local + +the call site cannot know which body it runs, so the direct call `methods` +measures is not available to it. every other object benchmark here has exactly +one candidate; this is the one that does not, which is the case a devirtualising +compiler has to get right and a direct-calling one has to get wrong loudly +""" + + +class Shape: + def __init__(self, size: int): + self.size = size + + def area(self) -> int: + return self.size + + +class Square(Shape): + def area(self) -> int: + return self.size * self.size + + +def total(shapes: list[Shape], passes: int) -> int: + running = 0 + p = 0 + while p < passes: + i = 0 + while i < len(shapes): + running = running + shapes[i].area() + i = i + 1 + p = p + 1 + return running + + +def bench() -> int: + shapes: list[Shape] = [] + i = 0 + while i < 200: + if i % 2 == 0: + shapes.append(Shape(i)) + else: + shapes.append(Square(i)) + i = i + 1 + return total(shapes, 300) diff --git a/scripts/native-bench/programs/keybuild.py b/scripts/native-bench/programs/keybuild.py new file mode 100644 index 0000000000..1efa72e6ef --- /dev/null +++ b/scripts/native-bench/programs/keybuild.py @@ -0,0 +1,22 @@ +"""`"k" + str(i)`: a builtin through the module namespace, a call, and a concat + +there is no dict here at all. this expression sat in every loop of the dict +benchmarks and was most of what they measured, so it is timed on its own and +they hand their keys in already built. `len` is called once, at the end +""" + + +def keys(n: int, passes: int) -> int: + last = "k" + p = 0 + while p < passes: + i = 0 + while i < n: + last = "k" + str(i) + i = i + 1 + p = p + 1 + return len(last) + + +def bench() -> int: + return keys(2000, 25) diff --git a/scripts/native-bench/programs/loops.py b/scripts/native-bench/programs/loops.py new file mode 100644 index 0000000000..cb6a7d0de4 --- /dev/null +++ b/scripts/native-bench/programs/loops.py @@ -0,0 +1,30 @@ +"""tagged-integer arithmetic with no container, no call and no float + +every other integer benchmark here carries something else: `sieve` owns a list, +`mandel` is floats, `recurse` is calls. this one is the arithmetic on its own, +so it is the floor a compiled loop can be measured against +""" + + +def collatz(n: int) -> int: + steps = 0 + while n != 1: + if n % 2 == 0: + n = n // 2 + else: + n = 3 * n + 1 + steps = steps + 1 + return steps + + +def total(limit: int) -> int: + running = 0 + i = 1 + while i < limit: + running = running + collatz(i) + i = i + 1 + return running + + +def bench() -> int: + return total(6000) diff --git a/scripts/native-bench/programs/mandel.py b/scripts/native-bench/programs/mandel.py new file mode 100644 index 0000000000..96eb6d39f6 --- /dev/null +++ b/scripts/native-bench/programs/mandel.py @@ -0,0 +1,38 @@ +"""scalar float arithmetic with a call in the inner loop + +the pair with `mandel_inline` is the measurement: same work, one with the escape +loop behind a call and one with it spelled out, so the difference is what a call +costs on the hot path +""" + + +def escape(cr: float, ci: float, limit: int) -> int: + zr = 0.0 + zi = 0.0 + k = 0 + while k < limit: + if zr * zr + zi * zi > 4.0: + return k + t = zr * zr - zi * zi + cr + zi = 2.0 * zr * zi + ci + zr = t + k = k + 1 + return limit + + +def render(width: int, height: int, limit: int) -> int: + total = 0 + y = 0 + while y < height: + x = 0 + while x < width: + cr = -2.0 + 3.0 * x / width + ci = -1.2 + 2.4 * y / height + total = total + escape(cr, ci, limit) + x = x + 1 + y = y + 1 + return total + + +def bench() -> int: + return render(120, 120, 40) diff --git a/scripts/native-bench/programs/mandel_inline.py b/scripts/native-bench/programs/mandel_inline.py new file mode 100644 index 0000000000..a1c006964d --- /dev/null +++ b/scripts/native-bench/programs/mandel_inline.py @@ -0,0 +1,35 @@ +"""the same work as `mandel`, with the escape loop inlined + +the shape a nested loop takes when nobody factored the inner one out: the outer +body's locals feed the inner arithmetic and the inner loop leaves by `break`. +this once made the type checker not converge at all, so it is a canary as well +as a benchmark +""" + + +def render(width: int, height: int, limit: int) -> int: + total = 0 + y = 0 + while y < height: + x = 0 + while x < width: + cr = -2.0 + 3.0 * x / width + ci = -1.2 + 2.4 * y / height + zr = 0.0 + zi = 0.0 + k = 0 + while k < limit: + if zr * zr + zi * zi > 4.0: + break + t = zr * zr - zi * zi + cr + zi = 2.0 * zr * zi + ci + zr = t + k = k + 1 + total = total + k + x = x + 1 + y = y + 1 + return total + + +def bench() -> int: + return render(120, 120, 40) diff --git a/scripts/native-bench/programs/methods.py b/scripts/native-bench/programs/methods.py new file mode 100644 index 0000000000..c6b6b24c4f --- /dev/null +++ b/scripts/native-bench/programs/methods.py @@ -0,0 +1,28 @@ +"""a method call on a long-lived object, with the object built once + +the companion to `calls`: same trivial body, reached through an instance rather +than through a module global. the difference between the two is what the method +lookup costs, and neither of them allocates — `alloc` is where allocation is +measured +""" + + +class Counter: + def __init__(self, base: int): + self.base = base + + def step(self, k: int) -> int: + return self.base + k + + +def run(counter: Counter, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + counter.step(i) + i = i + 1 + return total + + +def bench() -> int: + return run(Counter(3), 300000) diff --git a/scripts/native-bench/programs/objects.py b/scripts/native-bench/programs/objects.py new file mode 100644 index 0000000000..d1ccf5d27f --- /dev/null +++ b/scripts/native-bench/programs/objects.py @@ -0,0 +1,30 @@ +"""a short-lived object that is then used: allocation, fields and methods together + +deliberately the mixture, because that is what object-shaped code looks like. +`alloc`, `fields` and `methods` are its three parts measured apart, and a change +that moves this one without moving any of those is worth explaining +""" + + +class Vec: + def __init__(self, x: float, y: float): + self.x = x + self.y = y + + def norm2(self) -> float: + return self.x * self.x + self.y * self.y + + def shift(self, dx: float, dy: float) -> float: + self.x = self.x + dx + self.y = self.y + dy + return self.x + self.y + + +def bench() -> float: + total = 0.0 + i = 0 + while i < 200000: + v = Vec(i * 0.5, i * 0.25) + total = total + v.norm2() + v.shift(1.0, 2.0) + i = i + 1 + return total diff --git a/scripts/native-bench/programs/prefix.py b/scripts/native-bench/programs/prefix.py new file mode 100644 index 0000000000..860f0626b3 --- /dev/null +++ b/scripts/native-bench/programs/prefix.py @@ -0,0 +1,29 @@ +"""a running sum appended into a list the function owns + +the growth case, and the only benchmark here that iterates a list with `for` +rather than by index — the two are different lowerings and `dot` covers the +other one +""" + + +def prefix(xs: list[float]) -> float: + out = [] + running = 0.0 + for x in xs: + running = running + x + out.append(running) + return out[len(out) - 1] + + +def bench() -> float: + xs = [] + i = 0 + while i < 100000: + xs.append(i * 0.001) + i = i + 1 + total = 0.0 + r = 0 + while r < 5: + total = total + prefix(xs) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/recurse.py b/scripts/native-bench/programs/recurse.py new file mode 100644 index 0000000000..28ac8a8ac2 --- /dev/null +++ b/scripts/native-bench/programs/recurse.py @@ -0,0 +1,16 @@ +"""calls that nest rather than repeat + +`calls` measures a call from a loop, where the caller's frame is reused all the +way down. this one measures depth: every call is live while the next is made, so +it is about the stack the compiler builds rather than about the call sequence +""" + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def bench() -> int: + return fib(24) diff --git a/scripts/native-bench/programs/sets.py b/scripts/native-bench/programs/sets.py new file mode 100644 index 0000000000..7ef4fc08e4 --- /dev/null +++ b/scripts/native-bench/programs/sets.py @@ -0,0 +1,33 @@ +"""a set built once, then tested for membership far more often than it grew + +the hash container without a value, which `dict` does not stand in for: the +membership test is the whole operation rather than a step before a read. the +shape is a short build and a long test phase, so the answer is about `in` +""" + + +def build(n: int) -> set[int]: + seen: set[int] = set() + i = 0 + while i < n: + seen.add(i * 3) + i = i + 1 + return seen + + +def hits(seen: set[int], n: int, passes: int) -> int: + found = 0 + p = 0 + while p < passes: + i = 0 + while i < n: + if i in seen: + found = found + 1 + i = i + 1 + p = p + 1 + return found + + +def bench() -> int: + seen = build(2000) + return hits(seen, 2000, 30) diff --git a/scripts/native-bench/programs/sieve.py b/scripts/native-bench/programs/sieve.py new file mode 100644 index 0000000000..8032ec2145 --- /dev/null +++ b/scripts/native-bench/programs/sieve.py @@ -0,0 +1,30 @@ +"""a sieve: a `bool` buffer the function owns, written from a nested loop + +indexed *writes*, which `dot` and `prefix` between them do not cover — and a +list of `bool`, which is the element type a compiler is most tempted to pack and +most likely to get wrong +""" + + +def sieve(limit: int) -> int: + flags = [] + i = 0 + while i < limit: + flags.append(True) + i = i + 1 + + count = 0 + n = 2 + while n < limit: + if flags[n]: + count = count + 1 + m = n + n + while m < limit: + flags[m] = False + m = m + n + n = n + 1 + return count + + +def bench() -> int: + return sieve(120000) diff --git a/scripts/native-bench/programs/strops.py b/scripts/native-bench/programs/strops.py new file mode 100644 index 0000000000..e360e4df57 --- /dev/null +++ b/scripts/native-bench/programs/strops.py @@ -0,0 +1,29 @@ +"""the string methods real code calls: `split`, `join`, `startswith`, `upper` + +`words` concatenates and `chars` indexes; between them they cover neither of the +two things python code actually does to a string, which is to take it apart and +put it back together through methods on `str` +""" + + +def normalise(line: str) -> int: + parts = line.split(" ") + kept = [] + for part in parts: + if part.startswith("w"): + kept.append(part.upper()) + return len("-".join(kept)) + + +def run(line: str, passes: int) -> int: + total = 0 + p = 0 + while p < passes: + total = total + normalise(line) + p = p + 1 + return total + + +def bench() -> int: + unit = "word0 word1 word2 zero3 word4 word5 zero6 word7 word8 word9" + return run(unit * 200, 60) diff --git a/scripts/native-bench/programs/tuples.py b/scripts/native-bench/programs/tuples.py new file mode 100644 index 0000000000..b1a5245932 --- /dev/null +++ b/scripts/native-bench/programs/tuples.py @@ -0,0 +1,25 @@ +"""tuples built, unpacked and indexed, and a function that returns two values + +the container real code reaches for when it wants a pair, and the only way a +python function returns more than one thing. a tuple is immutable and of known +length, which is a different problem from `list` and is not covered by it +""" + + +def split(value: int) -> tuple[int, int]: + return value // 7, value % 7 + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + whole, part = split(i) + pair = (whole, part) + total = total + pair[0] + pair[1] + i = i + 1 + return total + + +def bench() -> int: + return run(300000) diff --git a/scripts/native-bench/programs/words.py b/scripts/native-bench/programs/words.py new file mode 100644 index 0000000000..4a0420eab6 --- /dev/null +++ b/scripts/native-bench/programs/words.py @@ -0,0 +1,19 @@ +"""string concatenation in a loop, and nothing else + +building and scanning are separate problems with separate answers, so `chars` +scans a string of the same shape and this one only builds. measuring them +together hides whichever is cheaper +""" + + +def build(n: int) -> str: + out = "" + i = 0 + while i < n: + out = out + "word" + str(i % 10) + " " + i = i + 1 + return out + + +def bench() -> int: + return len(build(20000)) diff --git a/scripts/native-bench/timer.py b/scripts/native-bench/timer.py new file mode 100644 index 0000000000..398f247551 --- /dev/null +++ b/scripts/native-bench/timer.py @@ -0,0 +1,165 @@ +"""the timed half of the native benchmark suite, run under the target interpreter + +`bench.py` stages and builds; this imports what it built and times it. the two +are separate programs because everything here has to happen inside one process: +the whole method rests on the four builds of a benchmark being timed in the same +process, in the same wall-clock window, round by round, so that a load spike +lands on all of them rather than on whichever one happened to be running. + +that is possible at all only because each build is staged under a module name of +its own — `mandel_cpython`, `mandel_by`, `mandel_control`, `mandel_mypyc` — since +an extension module's init hook is found by name and two of them cannot answer to +the same one. + +nothing here is timed until it has proved what it is. `load` refuses a module +whose file is not under this run's own root, is older than this run's build, or — +for a build that is supposed to be compiled — does not end in a real extension +suffix. a stale artefact and a build that silently did not happen are the two +ways this suite has lied before, and both are refusals rather than warnings. +""" + +from __future__ import annotations + +import gc +import importlib +import importlib.machinery +import json +import math +import pathlib +import sys +import time + +# the interpreter puts *this script's* directory first, and a benchmark that +# shared a name with anything beside it would be shadowed. nothing is imported +# from here, so the entry only costs correctness +sys.path.pop(0) + + +class Refused(Exception): + """a build that cannot be proved to be the one that was just made""" + + +def load(leg: dict, root: str, built_after: float): + """import one build, or refuse to""" + sys.path.insert(0, leg["dir"]) + try: + module = importlib.import_module(leg["module"]) + except Exception as error: + raise Refused(f"import failed: {type(error).__name__}: {error}") from error + + origin = getattr(module, "__file__", None) + if origin is None: + raise Refused("the module has no __file__, so what ran cannot be identified") + path = pathlib.Path(origin).resolve() + + if not path.is_relative_to(pathlib.Path(root).resolve()): + raise Refused(f"imported {path}, which is outside this run's root") + if path.stat().st_mtime < built_after: + raise Refused(f"imported {path.name}, which predates this run's build") + + suffixes = tuple(importlib.machinery.EXTENSION_SUFFIXES) + if leg["compiled"]: + if not path.name.endswith(suffixes): + raise Refused(f"imported {path.name}, which is not an extension module") + elif path.name.endswith(suffixes): + raise Refused(f"imported {path.name}, which is an extension module") + + if not hasattr(module, "bench"): + raise Refused("the module has no bench()") + return module, str(path) + + +def sample(module, count: int) -> float: + """one timed sample: `count` calls, with the collector quiesced first + + `gc.collect()` sits outside the timed region rather than being disabled, + because disabling it would change what an allocation-heavy benchmark + measures. quiescing it only moves the collection that was going to happen + anyway out of whichever build was unlucky enough to trigger it + """ + gc.collect() + start = time.perf_counter() + for _ in range(count): + module.bench() + return time.perf_counter() - start + + +def main() -> int: + spec = json.loads(pathlib.Path(sys.argv[1]).read_text()) + root, built_after = spec["root"], spec["built_after"] + + loaded, refused, answers, origins = [], {}, {}, {} + for leg in spec["legs"]: + try: + module, origin = load(leg, root, built_after) + except Refused as refusal: + refused[leg["name"]] = str(refusal) + continue + loaded.append((leg["name"], module)) + origins[leg["name"]] = origin + + if spec["mode"] == "probe": + for name, module in loaded: + try: + answers[name] = repr(module.bench()) + except Exception as error: + refused[name] = f"bench() raised {type(error).__name__}: {error}" + print(json.dumps({"answers": answers, "refused": refused, "origins": origins})) + return 0 + + if spec["mode"] == "calibrate": + # how many calls make one sample. every build of a benchmark runs the + # same number, so the pairing stays exact — which means one number has + # to suit builds that can be forty times apart. it is chosen from both + # ends: long enough that the fastest build's sample is well clear of the + # clock, short enough that the slowest build's round does not dominate + # the run. where the two disagree the cap wins, and the control column + # then says out loud whether the resulting sample was too short + per_call = {} + for name, module in loaded: + count = 1 + while True: + elapsed = sample(module, count) + if elapsed >= spec["probe_target"] or count >= spec["max_count"]: + break + grow = max(2, min(8, int(spec["probe_target"] / max(elapsed, 1e-9)))) + count = min(count * grow, spec["max_count"]) + per_call[name] = elapsed / count + + fastest, slowest = min(per_call.values()), max(per_call.values()) + wanted = math.ceil(spec["min_sample"] / fastest) + cap = max(1, int(spec["max_sample"] / slowest)) + count = max(1, min(wanted, cap, spec["max_count"])) + print(json.dumps({"count": count, "per_call": per_call, "refused": refused})) + return 0 + + count, rounds, warmup = spec["count"], spec["rounds"], spec["warmup"] + order = [name for name, _ in loaded] + modules = dict(loaded) + timings = {name: [] for name in order} + + for index in range(warmup + rounds): + # rotate, so no build is systematically the one that pays for a cold + # cache at the top of a round. deterministic rather than random: a + # benchmark run should be reproducible + shift = index % len(order) + for name in order[shift:] + order[:shift]: + elapsed = sample(modules[name], count) + if index >= warmup: + timings[name].append(elapsed / count) + + print( + json.dumps( + { + "timings": timings, + "refused": refused, + "origins": origins, + "count": count, + } + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/native-sweeps/buildsweep.sh b/scripts/native-sweeps/buildsweep.sh index e63175ac77..c9f385ff1b 100755 --- a/scripts/native-sweeps/buildsweep.sh +++ b/scripts/native-sweeps/buildsweep.sh @@ -15,8 +15,8 @@ trap 'rm -rf "$root"' EXIT for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - err=$(cd "$d" && PYTHON="$PY" "$BY" compile m.py -o o 2>&1 >/dev/null) + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + err=$(cd "$d" && PYTHON="$PY" "$BY" compile "$SWEEP_SRC" -o o 2>&1 >/dev/null) if echo "$err" | grep -qiE 'panicked|internal error|stack overflow'; then printf '%s\tPANIC\t%s\n' "$b" "$(echo "$err" | head -1)" >> "$OUT" elif sweep_built "$d"; then diff --git a/scripts/native-sweeps/census.sh b/scripts/native-sweeps/census.sh new file mode 100755 index 0000000000..6a3f3d972f --- /dev/null +++ b/scripts/native-sweeps/census.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# compile every module with --annotate and collect the whole report per module +# usage: census.sh SP BY PY OUT [MODULE...] +SP="$1"; BY="$2"; PY="$3"; OUT="$4"; shift 4 +# shellcheck source=scripts/native-sweeps/sweeplib.sh +. "$(dirname "$0")/sweeplib.sh" +LIB=$(sweep_lib "$PY") +root="$SP/census.$$"; rm -rf "$root"; mkdir -p "$root" +trap 'rm -rf "$root"' EXIT +: > "$OUT" +for b in $(sweep_modules "$LIB" "$@"); do + f="$LIB/$b" + [ -f "$f" ] || continue + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" --annotate --emit-c-only + report="$(sweep_out_dir "$d")/m.annotated" + if [ ! -f "$report" ]; then printf '%s\tno-report\n' "$b" >> "$OUT"; continue; fi + awk -v b="$b" ' + /^[0-9]+ compiled, [0-9]+ left interpreted$/ { print b "\tcounts\t" $1 "\t" $3 } + /^- / && insec==1 { sub(/^- /,""); print b "\tdecline\t" $0 } + /^## left to the interpreted definition/ { insec=1; next } + /^## / && !/^## left to the interpreted definition/ { insec=0 } + /^## class / { print b "\tclass\t" $3 } + ' "$report" >> "$OUT" +done diff --git a/scripts/native-sweeps/instancecensus.sh b/scripts/native-sweeps/instancecensus.sh index 67b0b59de9..fdd3dcf8d8 100755 --- a/scripts/native-sweeps/instancecensus.sh +++ b/scripts/native-sweeps/instancecensus.sh @@ -25,9 +25,20 @@ trap 'rm -rf "$root"' EXIT : > "$OUT" cat > "$root/probe.py" <<'PYEOF' +import importlib +import os import signal import sys +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where an +# interpreted one answers `pkg.m`. this census reads a compiled leg only, and both +# spellings mean "defined by the module under test" +SELF = (MOD, MOD.rpartition('.')[2]) + def _ring(signum, frame): raise TimeoutError('timed out') @@ -35,9 +46,9 @@ def _ring(signum, frame): signal.signal(signal.SIGALRM, _ring) try: - signal.alarm(30) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - import m + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: @@ -51,7 +62,7 @@ counts = [0, 0] def note(value, bucket, slot): kind = type(value) - if getattr(kind, '__module__', None) != 'm': + if getattr(kind, '__module__', None) not in SELF: return claimed = getattr(m, kind.__name__, None) if not isinstance(claimed, type): @@ -99,19 +110,24 @@ PYEOF for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi # the swap this defect is about, read off the emitted C: every class whose name the # module init rebinds to a compiled type - emitted=$(grep -oE 'PyDict_SetItemString\(dict, "[^"]+", By_[A-Za-z0-9_]+_OBJ\)' "$d/o/m.c" \ + # + # read *before* `sweep_place`: a package member's C sits inside the package's place in + # the output tree, and `sweep_place` lays the twin's copy of the package over the top + emitted=$(grep -oE 'PyDict_SetItemString\(dict, "[^"]+", By_[A-Za-z0-9_]+_OBJ\)' \ + "$(sweep_out_dir "$d")/m.c" \ | sed -E 's/.*dict, "([^"]+)".*/\1/' | LC_ALL=C sort -u | paste -sd, -) - cp "$root/probe.py" "$d/o/probe.py" + sweep_place "$d" + cp "$root/probe.py" "$SWEEP_RUN_C/probe.py" # the comma list becomes one argument per class; an array says that, where a bare # command substitution only word-splits by accident names=() [ -n "$emitted" ] && IFS=',' read -r -a names <<< "$emitted" - out=$(cd "$d/o" && "$PY" probe.py "${names[@]}" 2>&1) + out=$(cd "$SWEEP_RUN_C" && "$PY" probe.py "${names[@]}" 2>&1) case "$out" in IMPORT-FAILED*) printf '%s\timport-failed\temitted=%s\n' "$b" "$emitted" >> "$OUT"; continue ;; esac diff --git a/scripts/native-sweeps/isoconstruct.sh b/scripts/native-sweeps/isoconstruct.sh index f3ccd352ea..55a6daaeca 100755 --- a/scripts/native-sweeps/isoconstruct.sh +++ b/scripts/native-sweeps/isoconstruct.sh @@ -23,53 +23,77 @@ trap 'rm -rf "$root"' EXIT cat > "$root/drive.py" <<'PYEOF' # every class the module itself defines, called with no arguments. `signal.alarm` # bounds a constructor that blocks, so one module cannot stall the sweep +import importlib +import os import signal import sys +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings mean "defined by the module under +# test", and nothing else in the staged tree answers to either +SELF = (MOD, MOD.rpartition('.')[2]) + + class _Slow(Exception): pass + def _ring(signum, frame): raise _Slow('timed out') -signal.signal(signal.SIGALRM, _ring) -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. so the failure is caught -# and reported as one line that carries no path at all -try: - signal.alarm(30) - try: - import m - finally: - signal.alarm(0) -except BaseException as error: - print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) - raise SystemExit(0) - -names = [ - name - for name in sorted(vars(m)) - if isinstance(vars(m)[name], type) and getattr(vars(m)[name], '__module__', None) == 'm' -] - -start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 - -for name in names[start:]: - value = vars(m)[name] +def main(): + signal.signal(signal.SIGALRM, _ring) + + # a module can still fail this import — a compiled extension that did not load, a + # C accelerator this build has no source for. both legs fail it alike, but an + # uncaught traceback would not read alike: the interpreted frame names `m.py` where + # the compiled one, running its fallback source through `PyRun_String`, names + # ``. so the failure is caught and reported as one line carrying no path try: - signal.alarm(2) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - made = value() + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: - print(name, type(error).__name__, str(error), flush=True) - else: - print(name, 'built', type(made).__name__, flush=True) + print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) + raise SystemExit(0) + + names = [ + name + for name in sorted(vars(m)) + if isinstance(vars(m)[name], type) + and getattr(vars(m)[name], '__module__', None) in SELF + ] + + start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + + for name in names[start:]: + value = vars(m)[name] + try: + signal.alarm(2) + try: + made = value() + finally: + signal.alarm(0) + except BaseException as error: + print(name, type(error).__name__, str(error), flush=True) + else: + print(name, 'built', type(made).__name__, flush=True) + + +# `multiprocessing` starts a worker by re-running this interpreter and importing this +# file, as `__mp_main__`. a constructor that makes a pool is reached now that the +# package modules import, and without the guard every worker would import the module +# and construct every class in it again — pool included +if __name__ == '__main__': + main() PYEOF # run one leg to completion, restarting past whatever killed it. a constructor that @@ -94,23 +118,31 @@ leg() { for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" # a traceback names the file it came from, and the two legs run from different # directories — so the *path* would read as a difference the module never had - i=$(leg "$d") - c=$(leg "$d/o") + i=$(leg "$SWEEP_RUN_I") + c=$(leg "$SWEEP_RUN_C") if [ "$i" = "$c" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(printf '%s' "$i" | grep -c '')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # a constructor that outran its two seconds on one leg and not the other, or an + # import that outran its thirty, says nothing about the compiler — a loaded machine + # loses either bound. kept out of `differing` so the headline number is the same on + # a busy machine as an idle one; it was not, and two agents reading the same tree + # got different counts + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(printf '%s' "$i") <(printf '%s' "$c") | awk -v b="$b" '{print b "\t| " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isoimport.sh b/scripts/native-sweeps/isoimport.sh index a402eb2690..2b1e1df01b 100755 --- a/scripts/native-sweeps/isoimport.sh +++ b/scripts/native-sweeps/isoimport.sh @@ -15,21 +15,51 @@ root="$SP/isoimp.$$"; rm -rf "$root"; mkdir -p "$root" trap 'rm -rf "$root"' EXIT : > "$OUT" -# an import that never returns would stall the whole sweep, and a subpackage module is -# far likelier to start something than a top-level one. the alarm is left at its default -# disposition on purpose: it kills the leg, both legs are killed alike, and a killed pair -# reads as `same` rather than as a difference nobody can act on -probe='import signal; signal.alarm(30); import m' +# an import that never returns would stall the whole sweep, and a package member is far +# likelier to start something than a top-level module. the alarm is left at its default +# disposition on purpose: it kills the leg, and a leg killed by it says nothing on the +# way out — so what tells the two cases apart is the *exit status*, not the output. +# +# a leg that dies prints nothing, and so does a leg that imports cleanly. reading only +# the text, this rung counted a killed leg as an import that worked — the one reading +# that must never be given, because it is agreement with a leg that answered nothing +# +# the name comes from the staging: `m` for a top-level module, `pkg.m` for a package +# member. both legs are handed the same one +probe='import importlib, os, signal; signal.alarm(int(os.environ["SWEEP_IMPORT_BOUND"])); importlib.import_module(os.environ["SWEEP_MOD"])' + +# one leg, into LEG_STATUS and LEG_TEXT. the status has to come from the interpreter +# rather than from the end of a pipeline, so the text is trimmed afterwards. a leg that +# says nothing is the case this exists for, so the two are returned apart rather than +# packed into one string an empty half would collapse +leg() { + local dir="$1" out + out=$(cd "$dir" && "$PY" -c "$probe" 2>&1); LEG_STATUS=$? + # the two legs run from different directories, so a message that names a file would + # read as a difference the module never had. each is made relative to its own root + LEG_TEXT=$(printf '%s' "$out" | tail -1 | sed "s|$dir/||g") +} for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - i=$(cd "$d" && "$PY" -c "$probe" 2>&1 | tail -1) - c=$(cd "$d/o" && "$PY" -c "$probe" 2>&1 | tail -1) - if [ "$i" = "$c" ]; then printf '%s\tsame\t%s\n' "$b" "$i" >> "$OUT" - else printf '%s\tDIFFERS\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$i" "$c" >> "$OUT"; fi + sweep_place "$d" + leg "$SWEEP_RUN_I"; istat=$LEG_STATUS; i=$LEG_TEXT + leg "$SWEEP_RUN_C"; cstat=$LEG_STATUS; c=$LEG_TEXT + # 142 is SIGALRM: the import outran the bound. a loaded machine loses it on one leg + # and not the other, which says nothing about the compiler — so it is its own + # category rather than a difference or an agreement + if [ "$istat" = 142 ] || [ "$cstat" = 142 ]; then + printf '%s\ttimed-out\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$istat" "$cstat" + elif [ "$istat" -gt 128 ] || [ "$cstat" -gt 128 ]; then + # killed by something else: a leg that died mid-import has not answered, and the + # empty text it leaves behind must not be read as agreement + printf '%s\tDIED\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$istat" "$cstat" + elif [ "$i" = "$c" ]; then printf '%s\tsame\t%s\n' "$b" "$i" + else printf '%s\tDIFFERS\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$i" "$c" + fi >> "$OUT" done -echo "walked: $(wc -l < "$OUT") exercised: $(grep -c $'\tsame\t$' "$OUT") differing: $(grep -c DIFFERS "$OUT") import-failed: $(grep -cE $'\tsame\t.' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(wc -l < "$OUT") exercised: $(grep -c $'\tsame\t$' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") died: $(grep -c $'\tDIED' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -cE $'\tsame\t.' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isosubclass.sh b/scripts/native-sweeps/isosubclass.sh index b14a7b1444..4a432a921a 100755 --- a/scripts/native-sweeps/isosubclass.sh +++ b/scripts/native-sweeps/isosubclass.sh @@ -29,10 +29,22 @@ trap 'rm -rf "$root"' EXIT cat > "$root/drive.py" <<'PYEOF' # every class the module itself defines, derived from. `signal.alarm` bounds a # metaclass that blocks, so one module cannot stall the sweep +import importlib +import os import signal import sys import types +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings mean "defined by the module under +# test", and nothing else in the staged tree answers to either +SELF = (MOD, MOD.rpartition('.')[2]) + class _Slow(Exception): pass @@ -42,51 +54,59 @@ def _ring(signum, frame): raise _Slow('timed out') -signal.signal(signal.SIGALRM, _ring) - -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. and a non-zero exit would -# send the restart loop below round forty times for nothing. so the failure is caught -# and reported as one line that carries no path at all -try: - signal.alarm(30) - try: - import m - finally: - signal.alarm(0) -except BaseException as error: - print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) - raise SystemExit(0) - -start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 -names = [ - name - for name in sorted(vars(m)) - if isinstance(vars(m)[name], type) and getattr(vars(m)[name], '__module__', None) == 'm' -] - - def _body(namespace): # a member the metaclass may want to do something with, which is what reaches # the descriptor protocol at all namespace['probe'] = 1 -for name in names[start:]: - value = vars(m)[name] +def main(): + signal.signal(signal.SIGALRM, _ring) + + # a module can still fail this import — a compiled extension that did not load, a + # C accelerator this build has no source for. both legs fail it alike, but an + # uncaught traceback would not read alike: the interpreted frame names `m.py` where + # the compiled one, running its fallback source through `PyRun_String`, names + # ``. and a non-zero exit would send the restart loop below round forty + # times for nothing. so the failure is caught and reported as one line carrying no + # path at all try: - signal.alarm(2) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - made = types.new_class('Sub', (value,), {}, _body) + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: - print(name, type(error).__name__, str(error), flush=True) - else: - print(name, 'subclassed', made.__name__, flush=True) + print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) + raise SystemExit(0) + + start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + names = [ + name + for name in sorted(vars(m)) + if isinstance(vars(m)[name], type) + and getattr(vars(m)[name], '__module__', None) in SELF + ] + + for name in names[start:]: + value = vars(m)[name] + try: + signal.alarm(2) + try: + made = types.new_class('Sub', (value,), {}, _body) + finally: + signal.alarm(0) + except BaseException as error: + print(name, type(error).__name__, str(error), flush=True) + else: + print(name, 'subclassed', made.__name__, flush=True) + + +# `multiprocessing` starts a worker by re-running this interpreter and importing this +# file, as `__mp_main__`. the package modules import now, so a module body that starts +# one is reachable, and without the guard the worker would walk the module again +if __name__ == '__main__': + main() PYEOF # run one leg to completion, restarting past whatever killed it @@ -112,21 +132,29 @@ leg() { for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" - i=$(leg "$d") - c=$(leg "$d/o") + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" + i=$(leg "$SWEEP_RUN_I") + c=$(leg "$SWEEP_RUN_C") if [ "$i" = "$c" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(printf '%s' "$i" | grep -c '')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # a class the metaclass took more than two seconds over, or an import that outran + # its thirty — a loaded machine loses either bound on one leg and not the other, + # which says nothing about the compiler. kept out of `differing` so the headline + # number is the same on a busy machine as an idle one, and out of `same` because a + # leg that was killed answered nothing + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(printf '%s' "$i") <(printf '%s' "$c") | awk -v b="$b" '{print b "\t| " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isosurface.sh b/scripts/native-sweeps/isosurface.sh index fb90df22f9..44ee5d233b 100755 --- a/scripts/native-sweeps/isosurface.sh +++ b/scripts/native-sweeps/isosurface.sh @@ -23,6 +23,11 @@ # - every module-level value whose type this module owns is asked whether it is still # an instance of the class now under that name # +# one cause currently dominates `differing`: an emitted class in a package reports its +# `__module__` without the package, so 215 of the 246 differ on nothing else. that is one +# defect, not 215 — `grep '| > .* named ' | grep -v \\.` separates them, and the rest is +# the number to watch +# # usage: isosurface.sh SP BY PY OUT [MODULE...] SP="$1"; BY="$2"; PY="$3"; OUT="$4"; shift 4 # shellcheck source=scripts/native-sweeps/sweeplib.sh @@ -33,8 +38,22 @@ trap 'rm -rf "$root"' EXIT : > "$OUT" cat > "$root/drive.py" <<'PYEOF' +import importlib +import os import signal +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings have to select the *same classes*, or +# the compiled leg would appear to define none of them. what a class says its module is +# stays compared outright, because that answer is wrong rather than merely differently +# spelled +SELF = (MOD, MOD.rpartition('.')[2]) + class _Slow(Exception): pass @@ -46,16 +65,15 @@ def _ring(signum, frame): signal.signal(signal.SIGALRM, _ring) -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. so the failure is caught -# and reported as one line that carries no path at all +# a module can still fail this import — a compiled extension that did not load, a C +# accelerator this build has no source for. both legs fail it alike, but an uncaught +# traceback would not read alike: the interpreted frame names `m.py` where the compiled +# one, running its fallback source through `PyRun_String`, names ``. so the +# failure is caught and reported as one line that carries no path at all try: - signal.alarm(30) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - import m + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: @@ -82,7 +100,7 @@ def value_of(target, key): def owned(value): - return isinstance(value, type) and getattr(value, '__module__', None) == 'm' + return isinstance(value, type) and getattr(value, '__module__', None) in SELF # what a compiled type does not carry, by construction rather than by defect: a spec @@ -101,6 +119,10 @@ for name, cls in classes: try: show('%s mro %s' % (name, [base.__name__ for base in cls.__mro__])) show('%s meta %s' % (name, type(cls).__name__)) + # `__module__` is compared outright, and the two spellings above are *not* + # collapsed here. a compiled class in a package answers `m` where its twin + # answers `pkg.m`, and that is a wrong answer rather than a house style: + # `dataclasses` does `sys.modules[cls.__module__].__dict__` and gets `None` show('%s named %s %s' % (name, cls.__module__, cls.__qualname__)) # one way: a name the interpreted class has and the compiled one does not. # `MISSING` is printed by the compiled leg only, so an identical pair of legs @@ -127,7 +149,7 @@ for name in sorted(vars(m)): if isinstance(value, type): continue kind = type(value) - if getattr(kind, '__module__', None) != 'm': + if getattr(kind, '__module__', None) not in SELF: continue claimed = getattr(m, kind.__name__, None) if not isinstance(claimed, type): @@ -142,29 +164,35 @@ PYEOF for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" # a traceback names the file it came from, and the two legs run from different # directories — so the *path* would read as a difference the module never had - i=$(cd "$d" && "$PY" drive.py 2>&1 | sed "s|$d/||g") - c=$(cd "$d/o" && "$PY" drive.py 2>&1 | sed "s|$d/o/||g") + i=$(cd "$SWEEP_RUN_I" && "$PY" drive.py 2>&1 | sed "s|$SWEEP_RUN_I/||g") + c=$(cd "$SWEEP_RUN_C" && "$PY" drive.py 2>&1 | sed "s|$SWEEP_RUN_C/||g") # a `has` line is one-directional: only its loss counts, so the compiled leg's extra # names are dropped before the comparison and the interpreted leg's are kept ionly=$(echo "$i" | grep -v ' has ') only=$(echo "$c" | grep -v ' has ') lost=$(comm -23 <(echo "$i" | grep ' has ' | sort) <(echo "$c" | grep ' has ' | sort)) if [ "$ionly" = "$only" ] && [ -z "$lost" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(echo "$i" | wc -l | tr -d ' ')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # the import bound is 30 seconds and a loaded machine loses it on one leg and not + # the other. that says nothing about the compiler, so it is kept out of `differing` + # — and out of `same`, because a leg that was killed answered nothing + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(echo "$ionly") <(echo "$only") | awk -v b="$b" '{print b "\t| " $0}' echo "$lost" | awk -v b="$b" 'NF {print b "\t| lost " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") lost: $(grep -c $'\t| lost ' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") lost: $(grep -c $'\t| lost ' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/sweeplib.sh b/scripts/native-sweeps/sweeplib.sh index 14532f4441..7451368ec8 100755 --- a/scripts/native-sweeps/sweeplib.sh +++ b/scripts/native-sweeps/sweeplib.sh @@ -43,45 +43,246 @@ sweep_modules() { | LC_ALL=C sort } +# the modules whose type inference does not terminate today +# +# they are still walked — a rung that skipped them would stop noticing the day they +# are fixed — but on a short leash, because the full bound is 180s and re-proving a +# known hang once per rung costs a quarter hour of every sweep cycle +# +# it is empty. three were on this list on the strength of "produced no artefact", +# and the stale-entry alarm caught all three on its first real run: on +# `origin/main` as much as here, `bdb.py` overflows the stack in 5s, and +# `pickletools.py` and `profile.py` finish in 5s with a salsa panic reported as a +# diagnostic. none of those is a hang, and none of them needs a leash +# +# `ast.py` came off when the return type it grew a constructor a round was bounded; +# it now compiles in 2s. `turtle.py` came off when a tuple whose elements were the +# cycle's own marker stopped alternating between the widened and the marked form of +# itself; it now compiles in 24s +# +# `${VAR-default}` rather than `${VAR:-default}`: an explicitly empty list is a +# caller saying "treat every hang as new", which is how this gets tested +BY_KNOWN_HANGS=${BY_KNOWN_HANGS-""} + # compile one staged module, bounded # -# the compiler is unbounded and a module whose type inference does not terminate wedges -# a whole rung: `ast.py` alone held one run for 21 minutes with no progress. the five -# that do this today (`ast`, `bdb`, `pickletools`, `profile`, `turtle`) stopped -# terminating when the branch was rebased, and the cause is on main rather than here — -# so the rungs have to survive it rather than wait for it. a bounded module reports as -# `no-artifact`, which is what it is: the sweep learned nothing about it +# a module that has never terminated needs only long enough to confirm it still does +# not; anything else gets the full bound. the two cases that must never pass quietly: +# a module *not* on the list that hits its bound is a new hang, and a module *on* the +# list that finishes means the list is stale. both say so on stderr and land in +# `$OUT.alarms`, because a rung that swallows either stops being evidence # -# SWEEP_BOUND is seconds; 0 disables the bound +# SWEEP_BOUND is seconds (0 disables); SWEEP_BOUND_KNOWN is the short leash sweep_compile() { - local dir="$1" py="$2" by="$3" - local bound=${SWEEP_BOUND:-180} + local name="$1" dir="$2" py="$3" by="$4"; shift 4 + local bound=${SWEEP_BOUND:-180} known=0 + case " $BY_KNOWN_HANGS " in *" $name "*) known=1; bound=${SWEEP_BOUND_KNOWN:-15} ;; esac + if [ "$bound" -eq 0 ]; then - (cd "$dir" && PYTHON="$py" "$by" compile m.py -o o >/dev/null 2>&1) + (cd "$dir" && PYTHON="$py" "$by" compile "$SWEEP_SRC" -o o "$@" >/dev/null 2>&1) return $? fi - (cd "$dir" && PYTHON="$py" "$by" compile m.py -o o >/dev/null 2>&1) & + + (cd "$dir" && PYTHON="$py" "$by" compile "$SWEEP_SRC" -o o "$@" >/dev/null 2>&1) & local pid=$! - ( sleep "$bound"; kill -9 "$pid" 2>/dev/null ) & + # the watchdog gets its own descriptors, and its `sleep` is killed with it. + # killing only the subshell leaves the `sleep` orphaned holding the inherited + # stdout, so a rung that finished in a second did not close its pipe until the + # bound elapsed — a reader saw the summary 180s late for no reason at all + { sleep "$bound"; kill -9 "$pid" 2>/dev/null; } >/dev/null 2>&1 & local killer=$! wait "$pid" 2>/dev/null local rc=$? + pkill -9 -P "$killer" 2>/dev/null kill -9 "$killer" 2>/dev/null wait "$killer" 2>/dev/null + + # 137 is the kill; anything else is the compiler's own answer + if [ "$rc" -eq 137 ]; then + if [ "$known" -eq 0 ]; then + sweep_alarm "$name" "did not finish in ${bound}s — a hang that is not on BY_KNOWN_HANGS" + fi + elif [ "$known" -eq 1 ]; then + sweep_alarm "$name" "finished in under ${bound}s — it is on BY_KNOWN_HANGS and should come off" + fi return $rc } -# lay one module out as a project of its own. `m.py` regardless of where it came from, -# so an import of `m` names the same thing in both legs +# an alarm is not a result: it says the sweep itself learned something it cannot +# record in a row, so it goes where it will be seen rather than into the tsv +sweep_alarm() { + printf '!! sweep alarm: %s %s\n' "$1" "$2" >&2 + [ -n "${OUT:-}" ] && printf '%s\t%s\n' "$1" "$2" >> "$OUT.alarms" + return 0 +} + +# the package every staged tree sits inside +# +# a package member has to be staged *in its package* or its relative imports have +# nothing to resolve against. a copy laid out under the package's own name would be +# imported in place of the interpreter's own, and that goes wrong two ways: `encodings` +# is already in `sys.modules` before a probe starts, so `import encodings.m` searches +# the real stdlib and finds nothing; and a copy of `re` or `importlib` first on the path +# answers every later import in the process, the driver's as much as the module's. one +# outer package nobody else names keeps the copy reachable and the interpreter's own +# stdlib intact. it costs the module only a prefix on its `__name__` +SWEEP_WRAP=by_stage + +# nothing the sweep runs benefits from a cache, and a `.pyc` is one more file that +# could answer in place of the one the sweep staged +export PYTHONDONTWRITEBYTECODE=1 + +# how long a leg is given to import the module, in seconds +# +# it is there to stop a module body that never returns, and thirty seconds is far more +# than any of them needs — but not more than the *machine* can take. macos scans a +# freshly written `.so` the first time it is loaded, and every module in the sweep +# builds a new one: on an idle machine that is under a second, and on a loaded one it +# has been measured at twenty-five, with the process asleep for all of it. the same +# extension then loads in 0.04s. so on a busy machine raise this rather than reading the +# timeouts as results +export SWEEP_IMPORT_BOUND=${SWEEP_IMPORT_BOUND:-30} + +# true when every directory above the module is a package +# +# a directory holding python files is not necessarily one: `config-3.13-darwin` has no +# `__init__.py`, and its name is not even an identifier. a module under one of those is +# reached by path rather than by import and is staged on its own, which is what it is +sweep_in_package() { + local lib="$1" rel="$2" prefix="" + case "$rel" in */*) ;; *) return 1 ;; esac + local rest="${rel%/*}" + while [ -n "$rest" ]; do + prefix="$prefix${rest%%/*}" + [ -f "$lib/$prefix/__init__.py" ] || return 1 + case "$rest" in + */*) rest="${rest#*/}"; prefix="$prefix/" ;; + *) rest="" ;; + esac + done + return 0 +} + +# lay one module out as a project of its own, and say where the two legs run +# +# the module is `m.py` wherever it came from, so both legs import the same name and +# `m.c`, `m.annotated` and `m*.so` keep theirs. +# +# a top-level module is staged alone, as it always was. a package member is staged in a +# copy of its whole package, so `from . import x` resolves — for the compiler as much as +# for the two legs. the siblings are on disk but kept out of the project's *file set*: +# `by compile` compiles every source in the project it is run from, and building all 122 +# modules of `encodings` once per member of `encodings` is fifteen thousand builds +# +# the whole package rather than the part the module imports: which part that is, is the +# same import graph the sweep exists to test, and a stage that had to be right about it +# would fail in the direction that hides defects +# +# sets, for the caller: +# SWEEP_MOD the dotted module name both legs import +# SWEEP_SRC the staged source, relative to $dir +# SWEEP_RUN_I the directory the interpreted leg runs from +# SWEEP_RUN_C the directory the compiled leg runs from sweep_stage() { - local dir="$1" src="$2" - rm -rf "$dir"; mkdir -p "$dir"; cp "$src" "$dir/m.py" - printf '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n' > "$dir/pyproject.toml" + local dir="$1" lib="$2" rel="$3" limit="" + rm -rf "$dir" "$dir/o"; mkdir -p "$dir" + export SWEEP_RUN_I="$dir" SWEEP_RUN_C="$dir/o" + if ! sweep_in_package "$lib" "$rel"; then + export SWEEP_MOD=m SWEEP_SRC=m.py + cp "$lib/$rel" "$dir/m.py" + else + local pkg="${rel%%/*}" sub="${rel%/*}" dotted + dotted=$(printf '%s' "$sub" | tr / .) + export SWEEP_SRC="$SWEEP_WRAP/$sub/m.py" + export SWEEP_MOD="$SWEEP_WRAP.$dotted.m" + mkdir -p "$dir/$SWEEP_WRAP" + : > "$dir/$SWEEP_WRAP/__init__.py" + cp -R "$lib/$pkg" "$dir/$SWEEP_WRAP/$pkg" + find "$dir/$SWEEP_WRAP" -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null + cp "$lib/$rel" "$dir/$SWEEP_SRC" + # a few package names are in ty's default `src.exclude` — `venv` above all — so a + # member of one is dropped before anything can compile it, and the rung reports + # `no-artifact` for a module that is perfectly fine. the negation re-includes the + # directory; it is written per-stage rather than as a blanket rule so the exclusion + # still holds for everything the sweep did not deliberately stage + limit=$(printf '[tool.ty.src]\ninclude=["%s"]\nexclude=["!**/%s/"]\n' \ + "$SWEEP_SRC" "${rel%%/*}") + fi + printf '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n%s' "$limit" \ + > "$dir/pyproject.toml" +} + +# where the build wrote the staged module's artefacts +# +# `by compile` lays its output out as the *module* tree, so a package member's `m.c`, +# `m.annotated` and `m*.so` sit at the member's own place under `o` rather than at the +# top of it — which is where the staged source sits within the project +sweep_out_dir() { + local rel="${SWEEP_SRC%/*}" + if [ "$rel" = "$SWEEP_SRC" ]; then printf '%s/o' "$1"; else printf '%s/o/%s' "$1" "$rel"; fi +} + +# put the twin's sources around the extension the compiled leg will import +# +# a package member's compiled leg needs the tree its twin has, and the build has already +# left the extension in the module's place inside it — so the extension is set aside +# while the twin's copy of the package is laid down over the top, then put back. the +# staged `m.py` is *removed* from it: python prefers an extension to a source of the same +# name, so leaving it there would mean an extension that failed to load was replaced by +# the interpreted module — and the leg would then agree with its twin for the one reason +# that makes the whole comparison meaningless +# +# call it after `sweep_built`, before running either leg +# the extension the build left, or nothing when it left none +# +# a glob rather than `ls`: what comes back is a path this file goes on to `mv` and to hand +# to python, and `ls` would mangle any name that needed quoting. an unmatched glob stays +# literal in bash, so the `-e` is what distinguishes "no artefact" from "one named `m*.so`" +sweep_artifact() { + local matches + matches=("$(sweep_out_dir "$1")"/m*.so) + [ -e "${matches[0]}" ] || return 1 + printf '%s\n' "${matches[0]}" +} + +sweep_place() { + local dir="$1" so name + so=$(sweep_artifact "$dir") || so="" + if [ "$SWEEP_MOD" != m ]; then + name=${so##*/} + mv "$so" "$dir/$name" + rm -rf "${dir:?}/o/$SWEEP_WRAP" + cp -R "$dir/$SWEEP_WRAP" "$dir/o/$SWEEP_WRAP" + rm -f "$dir/o/$SWEEP_SRC" + so="$dir/o/${SWEEP_SRC%/*}/$name" + mv "$dir/$name" "$so" + fi + sweep_warm "$so" +} + +# pay the operating system's first-load cost before anything is timed +# +# macos validates a freshly built dylib the first time it is loaded, and the process is +# asleep for all of it: 0.42s on an idle machine against 0.04s for every load after, and +# measured at 17.8s under contention. every module in a sweep builds a new one, so that +# cost lands inside whatever bound the rung set and comes back as a `timed-out` — which +# reads as a result and is not one. a full `isoimport` reported 26 of them; re-running +# exactly those 26 with a larger bound gave 26 exercised and 0 differing, so it was the +# operating system every time +# +# `ctypes.CDLL` loads the object without calling `PyInit_`, so this pays the validation +# without running the module body. that matters: a warm-up that imported the module would +# run its import side effects an extra time, and one that hung would hang here instead +sweep_warm() { + local so="$1" + [ -n "$so" ] && [ -f "$so" ] || return 0 + "$PY" -c 'import ctypes, sys; ctypes.CDLL(sys.argv[1])' "$so" >/dev/null 2>&1 + return 0 } # true when the build actually left an extension module behind. `-d o` is not enough: # a build that fails halfway leaves the directory and no artefact, and the compiled leg # then fails to import for a reason that is not the defect the sweep is looking for sweep_built() { - ls "$1"/o/m*.so >/dev/null 2>&1 + sweep_artifact "$1" >/dev/null } diff --git a/scripts/shrink.py b/scripts/shrink.py new file mode 100755 index 0000000000..e295afa398 --- /dev/null +++ b/scripts/shrink.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""shrink a python file to the smallest one that still provokes a `by` failure + +every cycle panic fixed in this repo was cracked by minimising first: 2714 lines to 21, +5055 to 4, 2904 to 3, 621 to 4. in each case the mechanism guessed beforehand turned out +wrong, and the small reproducer is what settled it. + +usage: + scripts/shrink.py --by [--python ] + [--marker "too many cycle iterations"] [--command check] + +three traps guarded here, each of which produced a confident wrong answer before it was: + +1. every candidate gets its own directory. `by check`/`by compile` operate on the whole + *project*, so an original sitting beside a candidate is analysed together with it — + the predicate is then true no matter what the candidate says, and this "shrank" 2714 + lines to 4 that reproduce nothing at all. + +2. a candidate whose rendered source equals the current best is skipped. after a + successful removal the cached statement holders can still hold nodes now detached from + the tree; mutating those re-renders identical source, and the predicate "passes" + without anything having changed. one run reported 8982 hits against 16 misses with the + line count frozen. + +3. the predicate is checked against a known-failing *and* a known-clean input before any + shrinking starts. `timeout` does not exist on macos, so a script relying on it once + answered "OK" for everything. + +the tell for all three is the same: **a predicate that never says no is not minimising, +it is deleting**. the hit/miss ratio is printed for that reason — a healthy run is on the +order of 15 hits to 40 misses, not 8982 to 16. +""" + +from __future__ import annotations + +import argparse +import ast +import os +import shutil +import subprocess +import sys +import tempfile + +PYPROJECT = '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n' +BODY_FIELDS = ("body", "orelse", "finalbody") + + +class Predicate: + """does `by` still fail the way we are shrinking towards?""" + + def __init__(self, by, python, marker, command, suffix): + self.by, self.python = by, python + self.marker, self.command, self.suffix = marker, command, suffix + self.hits = self.misses = 0 + + def __call__(self, source): + # trap 1: its own directory, holding only this candidate + work = tempfile.mkdtemp(prefix="shrink-") + try: + name = f"m{self.suffix}" + with open(os.path.join(work, name), "w") as handle: + handle.write(source) + with open(os.path.join(work, "pyproject.toml"), "w") as handle: + handle.write(PYPROJECT) + env = {**os.environ, "PYTHON": self.python} + try: + done = subprocess.run( + [self.by, self.command, name], + cwd=work, + env=env, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.TimeoutExpired: + self.misses += 1 + return False + hit = self.marker in (done.stdout + done.stderr) + self.hits += hit + self.misses += not hit + return hit + finally: + shutil.rmtree(work, ignore_errors=True) + + +def holders(tree): + """every statement list in the tree, as (list, ) — one entry per body""" + found = [] + for node in ast.walk(tree): + for field in BODY_FIELDS: + body = getattr(node, field, None) + if ( + isinstance(body, list) + and body + and all(isinstance(s, ast.stmt) for s in body) + ): + found.append(body) + for handler in getattr(node, "handlers", None) or []: + found.append(handler.body) + return found + + +def shrink(source, still_fails): + """greedily drop statements at any depth while the predicate holds""" + best = source + changed = True + rounds = 0 + while changed: + changed, rounds = False, rounds + 1 + for index in range(len(holders(ast.parse(best)))): + at = 0 + while True: + tree = ast.parse(best) + bodies = holders(tree) + if index >= len(bodies) or at >= len(bodies[index]): + break + body = bodies[index] + body.pop(at) + if not body: + body.append(ast.Pass()) + try: + candidate = ast.unparse(tree) + except (ValueError, AttributeError, RecursionError): + at += 1 + continue + # trap 2: an identical rendering proves nothing, so it is not a trial + if candidate == best: + at += 1 + continue + if still_fails(candidate): + best = candidate # the list shifted down, so `at` stays put + changed = True + else: + at += 1 + print(f"round {rounds}: {len(best.splitlines())} lines", flush=True) + return best + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source") + parser.add_argument("output") + parser.add_argument("--by", required=True, help="a release-built `by`") + parser.add_argument("--python", default=sys.executable) + parser.add_argument("--marker", default="too many cycle iterations") + parser.add_argument( + "--command", default="check", help="check | compile | transpile" + ) + args = parser.parse_args() + + with open(args.source) as handle: + source = handle.read() + suffix = os.path.splitext(args.source)[1] or ".py" + still_fails = Predicate(args.by, args.python, args.marker, args.command, suffix) + + # trap 3: a predicate that cannot say no is not a predicate + if not still_fails(source): + sys.exit(f"the unmodified source does not produce {args.marker!r}") + if still_fails("x = 1\n"): + sys.exit( + "a trivial clean file also matches — the predicate is not discriminating" + ) + print("predicate validated on a failing and a clean input", flush=True) + + # normalise through `unparse` first, so the shrinker's own rendering is the baseline + # and the first successful removal is not credited to a formatting change + base = ast.unparse(ast.parse(source)) + if not still_fails(base): + sys.exit( + "the source stops failing once reformatted — shrink the original by hand" + ) + + best = shrink(base, still_fails) + with open(args.output, "w") as handle: + handle.write(best + "\n") + print( + f"final: {len(best.splitlines())} lines " + f"({still_fails.hits} hits / {still_fails.misses} misses)", + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/ty.schema.json b/ty.schema.json index 6e281b01aa..878b371c38 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -2454,7 +2454,7 @@ "type": "object", "properties": { "exclude": { - "description": "A list of file and directory patterns to exclude from type checking.\n\nPatterns follow a syntax similar to `.gitignore`:\n\n- `./src/` matches only a directory\n- `./src` matches both files and directories\n- `src` matches files or directories named `src`\n- `*` matches any (possibly empty) sequence of characters (except `/`).\n- `**` matches zero or more path components.\n This sequence **must** form a single path component, so both `**a` and `b**` are invalid and will result in an error.\n A sequence of more than two consecutive `*` characters is also invalid.\n- `?` matches any single character except `/`\n- `[abc]` matches any character inside the brackets. Character sequences can also specify ranges of characters, as ordered by Unicode,\n so e.g. `[0-9]` specifies any character between `0` and `9` inclusive. An unclosed bracket is invalid.\n- `!pattern` negates a pattern (undoes the exclusion of files that would otherwise be excluded)\n\nAll paths are anchored relative to the project root (`src` only\nmatches `/src` and not `/test/src`).\nTo exclude any directory or file named `src`, use `**/src` instead.\n\nBy default, ty excludes commonly ignored directories:\n\n- `**/.bzr/`\n- `**/.direnv/`\n- `**/.eggs/`\n- `**/.git/`\n- `**/.git-rewrite/`\n- `**/.hg/`\n- `**/.mypy_cache/`\n- `**/.nox/`\n- `**/.pants.d/`\n- `**/.pytype/`\n- `**/.ruff_cache/`\n- `**/.svn/`\n- `**/.tox/`\n- `**/.venv/`\n- `**/__pypackages__/`\n- `**/_build/`\n- `**/buck-out/`\n- `**/dist/`\n- `**/node_modules/`\n- `**/venv/`\n\nYou can override any default exclude by using a negated pattern. For example,\nto re-include `dist` use `exclude = [\"!dist\"]`", + "description": "A list of file and directory patterns to exclude from type checking.\n\nPatterns follow a syntax similar to `.gitignore`:\n\n- `./src/` matches only a directory\n- `./src` matches both files and directories\n- `src` matches files or directories named `src`\n- `*` matches any (possibly empty) sequence of characters (except `/`).\n- `**` matches zero or more path components.\n This sequence **must** form a single path component, so both `**a` and `b**` are invalid and will result in an error.\n A sequence of more than two consecutive `*` characters is also invalid.\n- `?` matches any single character except `/`\n- `[abc]` matches any character inside the brackets. Character sequences can also specify ranges of characters, as ordered by Unicode,\n so e.g. `[0-9]` specifies any character between `0` and `9` inclusive. An unclosed bracket is invalid.\n- `!pattern` negates a pattern (undoes the exclusion of files that would otherwise be excluded)\n\nAll paths are anchored relative to the project root (`src` only\nmatches `/src` and not `/test/src`).\nTo exclude any directory or file named `src`, use `**/src` instead.\n\nBy default, ty excludes commonly ignored directories:\n\n- `**/.bzr/`\n- `**/.direnv/`\n- `**/.eggs/`\n- `**/.git/`\n- `**/.git-rewrite/`\n- `**/.hg/`\n- `**/.mypy_cache/`\n- `**/.nox/`\n- `**/.pants.d/`\n- `**/.pytype/`\n- `**/.ruff_cache/`\n- `**/.svn/`\n- `**/.tox/`\n- `**/.venv/`\n- `**/__pypackages__/`\n- `**/_build/`\n- `**/buck-out/`\n- `**/dist/`\n- `**/node_modules/`\n- `**/venv/`\n\nYou can override any default exclude by using a negated pattern. For example,\nto re-include `dist` use `exclude = [\"!dist\"]`, or `exclude = [\"!**/dist/\"]` to\nre-include every `dist` directory rather than only the one at the project root.\n\nA negated pattern can only re-include something that is still walked, so it cannot\nreach into a directory that is itself excluded. `exclude = [\"!dist/generated.py\"]`\nre-includes nothing, because the walk stops at `dist`. Re-include the directory\nfirst: `exclude = [\"!**/dist/\", \"**/dist/**\", \"!**/dist/generated.py\"]`", "anyOf": [ { "$ref": "#/definitions/Array_of_string" diff --git a/zensical.toml b/zensical.toml index 51d0b20cfc..4dbab0b31a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -220,6 +220,7 @@ development = [ "development/compilation/optimizations.md", "development/compilation/planned-features.md", "development/compilation/runtime.md", + "development/compilation/benchmarks.md", "development/compilation/plan.md", ] }, ]