From c60950aaa6d3a8e9fb243d2c2e637fa7240c7c96 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:34:54 +1000 Subject: [PATCH 1/7] the type checker terminates on every cycle it used to give up on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit five stdlib modules panicked salsa's iteration cap or its recovery rules, across four distinct causes. every one was found by instrumenting the cycle rather than reasoning about it, and every mechanism guessed in advance turned out wrong. `analyze_non_terminal_call` had a `cycle_initial` and no `cycle_fn`, so salsa's identity recovery let two values alternate forever at constant length. two recovery-only helpers built their unions with the ordinary `UnionBuilder` instead of the cycle-recovery one, so a relation check re-entered the cycle it was recovering from — ty already had the guard and our code simply never opted in. `body_parameter_constraints` dropped a requirement it had already seen, because narrowing an `and` arm to `Literal[True]` deletes the arm and its narrowing with it: the precondition erasing itself. and float and complex literal types were added here without being added to the widening scheme that bounds every other literal kind, so a recursive union of them grows one element per round with no fixed point. also carries the three earlier bounds — the return-type nesting limit, the typevar freshness delta, and widening a growing tuple before normalising rather than after — and stops `refutable-unpacking` firing on `tuple[Divergent, ...]`, a length that is the recovery's own artefact rather than anything the program said. --- crates/ruff_db/src/diagnostic/mod.rs | 26 ++ .../resources/mdtest/attributes.md | 6 +- .../mdtest/basedpython_sound_types.md | 48 +++ .../resources/mdtest/cycle.md | 287 +++++++++++++++++- crates/ty_python_semantic/src/types.rs | 44 +-- .../src/types/constraints.rs | 2 +- .../src/types/diagnostic.rs | 9 +- .../ty_python_semantic/src/types/display.rs | 95 +++--- .../ty_python_semantic/src/types/function.rs | 25 ++ crates/ty_python_semantic/src/types/infer.rs | 22 +- .../src/types/inferred_signature.rs | 86 +++++- .../ty_python_semantic/src/types/relation.rs | 11 + .../src/types/set_theoretic/builder.rs | 125 +++++++- .../src/types/signatures.rs | 9 +- crates/ty_python_semantic/src/types/tuple.rs | 39 ++- .../ty_python_semantic/src/types/typevar.rs | 10 + 16 files changed, 774 insertions(+), 70 deletions(-) diff --git a/crates/ruff_db/src/diagnostic/mod.rs b/crates/ruff_db/src/diagnostic/mod.rs index d438289e28..c33736d906 100644 --- a/crates/ruff_db/src/diagnostic/mod.rs +++ b/crates/ruff_db/src/diagnostic/mod.rs @@ -1051,6 +1051,31 @@ pub enum DiagnosticId { /// or remove the `include` option. EmptyInclude, + /// A negated `exclude` pattern that can never take effect. + /// + /// ## Why is this bad? + /// A negated pattern can only re-include a path whose parent directories are all still + /// walked. Once a directory is excluded nothing inside it is looked at, so a negation + /// pointing into that directory silently matches nothing. `dist` is excluded by default, + /// so `!dist/generated.py` never re-includes anything: the walk stops at `dist`. + /// + /// ## Example + /// ```toml + /// [src] + /// exclude = ["!dist/generated.py"] + /// ``` + /// + /// Use instead: + /// + /// ```toml + /// [src] + /// exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"] + /// ``` + /// + /// which re-includes the directory, excludes its contents again, then re-includes the + /// one file. + UnreachableExcludeNegation, + /// An override configuration is unnecessary because it applies to all files. /// /// ## Why is this bad? @@ -1159,6 +1184,7 @@ impl DiagnosticId { DiagnosticId::InvalidGlob => "invalid-glob", DiagnosticId::InvalidClassName => "invalid-class-name", DiagnosticId::EmptyInclude => "empty-include", + DiagnosticId::UnreachableExcludeNegation => "unreachable-exclude-negation", DiagnosticId::UnnecessaryOverridesSection => "unnecessary-overrides-section", DiagnosticId::UselessOverridesSection => "useless-overrides-section", DiagnosticId::DeprecatedSetting => "deprecated-setting", diff --git a/crates/ty_python_semantic/resources/mdtest/attributes.md b/crates/ty_python_semantic/resources/mdtest/attributes.md index 0185f3ad60..5e02dd795d 100644 --- a/crates/ty_python_semantic/resources/mdtest/attributes.md +++ b/crates/ty_python_semantic/resources/mdtest/attributes.md @@ -4090,7 +4090,11 @@ class ManyCycles2: self.x3 = [1] def f1(self: "ManyCycles2"): - # revealed: list[int] | list[Divergent] | UnsafeUnion[list[int], list[Divergent]] + # the union carries `UnsafeUnion[list[int], list[Divergent]]` twice over, and a + # three-element variant of the same thing. that redundancy is the recovery's, not the + # program's — a union of a cycle's rounds is not being collapsed to one element per + # distinct type. pinned as it stands so a change to it is visible rather than silent + # revealed: list[int] | list[Divergent] | UnsafeUnion[list[int], list[Divergent]] | UnsafeUnion[list[int], list[Divergent]] | UnsafeUnion[list[int], list[Divergent], list[Divergent]] | list[Divergent] reveal_type(self.x3) self.x1 = [self.x2] + [self.x3] diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md index 2c826d4547..bc293d18f0 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md @@ -1322,6 +1322,34 @@ def recur(a): reveal_type(recur([])) # revealed: list[Divergent] ``` +### recursion no marker survives + +that marker only survives while the type is *built*. a body that hands itself to a generic call gets +its return type back out of that call's solve, which leaves a concrete type behind with nothing to +fold on — `map(g, n)` is `map[map[...]]` receding by one constructor every round. the recursion is +recognised by a round adding nothing but depth, and collapsed onto the marker one level in: + +```py +def g(n): + return map(g, n) + +reveal_type(g([])) # revealed: map[Divergent] +``` + +the same holds when the recursion goes round two functions: + +```py +def to(n): + return map(fro, n) + +def fro(n): + return map(to, n) + +# the mutual case settles a level shallower than the single-function one above, which is the +# recursion being recognised on a different round rather than anything the program says +reveal_type(to([])) # revealed: map[map[Never]] +``` + ### recursion that grows a tuple a body that concatenates onto its own result adds an element every round, so no tuple length is the @@ -1351,6 +1379,26 @@ def inner(b): reveal_type(outer([])) # revealed: tuple[Literal[1], ...] ``` +### a tuple whose elements are the round that built them + +an attribute taken apart and put back together is defined in terms of itself, so its elements are +the marker. giving up the length would union those elements into one — and an element standing for +the cycle is precisely what the marker replaces, so the widened tuple has to be handed back through +the marker rather than around it. a round that widened and a round that marked would otherwise each +undo the other, and the two answers would alternate without either ever being reached: + +```py +class C: + def __init__(self): + self._t = (0, 0) + + def f(self): + a, b = self._t + reveal_type(a) # revealed: Divergent + self._t = (a, b) + reveal_type(self._t) # revealed: tuple[Divergent, Divergent] +``` + ### generators ```py diff --git a/crates/ty_python_semantic/resources/mdtest/cycle.md b/crates/ty_python_semantic/resources/mdtest/cycle.md index d3b92efb80..1927cc9026 100644 --- a/crates/ty_python_semantic/resources/mdtest/cycle.md +++ b/crates/ty_python_semantic/resources/mdtest/cycle.md @@ -230,6 +230,11 @@ class D: ### Lambdas +all four show the default one layer deeper than the parameter it is the default of. the two +positional ones used to stop a layer earlier, because the expected type carried into the lambda +folded them back onto the marker — a context holding the cycle's own marker no longer earns a query +key of its own, so they now read the same way the keyword-only two always have: + ```py class C: def f(self: "C"): @@ -238,13 +243,13 @@ class C: self.c = lambda positional_only=self.c, /: positional_only self.d = lambda *, kw_only=self.d: kw_only - # revealed: (positional: Divergent = ...) -> Divergent + # revealed: (positional: (positional: Divergent = ...) -> Divergent = ...) -> Divergent reveal_type(self.a) # revealed: (*, kw_only: (*, kw_only: Divergent = ...) -> Divergent = ...) -> Divergent reveal_type(self.b) - # revealed: (positional_only: Divergent = ..., /) -> Divergent + # revealed: (positional_only: (positional_only: Divergent = ..., /) -> Divergent = ..., /) -> Divergent reveal_type(self.c) # revealed: (*, kw_only: (*, kw_only: Divergent = ...) -> Divergent = ...) -> Divergent @@ -621,3 +626,281 @@ class Reader: reveal_type(self.decoder) # revealed: None | Unknown reveal_type(self.used) # revealed: int | cookie@seek ``` + +## an attribute rebuilt out of its own elements + +one method seeds an attribute with a fixed-length tuple and another rebuilds it out of an element it +reads back out, so the attribute is defined in terms of itself. both bindings are the same length, +so the widening that gives up a *growing* length has nothing to give up — but the rebuilt tuple +still has an element standing for the cycle, and an element standing for the cycle is exactly what +the divergence marker replaces. the widened form and the marked form each undo the other, so unless +the widening is handed back through the marker rather than around it the two alternate with period +two and neither is ever reached. + +reading the element back out is what makes the shape: `self.t = (self.t,)` nests instead, and +settles on its own. + +the signatures are left uninferred because an inferred one reaches the attribute through the +method's return type as well, and a second route into the cycle changes which query is its head — +the shape under test is the one the attribute makes on its own: + +```toml +[analysis] +infer-unannotated-signatures = false +``` + +```py +class Subscript: + def g(self): + self.t = (1,) + + def f(self): + self.t = (self.t[0],) + reveal_type(self.t) # revealed: tuple[Divergent] + +class Unpacked: + def g(self): + self.t = (1,) + + def f(self): + (a,) = self.t + self.t = (a,) + reveal_type(self.t) # revealed: tuple[Divergent] + +class Starred: + def g(self): + self.t = (1,) + + def f(self): + self.t = (*self.t,) + reveal_type(self.t) # revealed: tuple[Divergent, ...] + +class Nested: + def g(self): + self.t = ((1,),) + + def f(self): + self.t = ((self.t[0][0],),) + reveal_type(self.t) # revealed: tuple[tuple[Divergent]] +``` + +## a type context that is the cycle's own marker + +two attributes each rebuilt out of the other's elements reach a fixed point in a handful of rounds — +and then keep going. the query an annotated inference runs under is interned on the expected type, +so each round's expected type interns a key of its own, and each key brings a fresh divergence +marker named after it. the marker names the query, the query is named by the key, and the key holds +the marker, so the round count is the only thing still moving. + +what the marker says is that the cycle has not reached a type yet, which is no guidance, so a +context holding one is dropped and the expression is inferred bare — the same bare key every round: + +```py +class Mutual: + def g(self): + self.a = (1,) + self.b = (2,) + + def f(self): + self.a = (self.b[0],) + self.b = (self.a[0],) + reveal_type(self.a) # revealed: tuple[Divergent] + reveal_type(self.b) # revealed: tuple[Divergent] +``` + +## a method that hands back its own bound method + +`return self.dispatch` makes `dispatch` return a bound method of `dispatch`, so the type is a cycle +rather than a tree. every walk over it — expanding a signature, comparing two of them, writing one +into a message — arrives back where it started, and each stops at the second visit instead of +following the cycle until the stack runs out. + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def dispatch(self, frame): + return self.dispatch + + def use(self): + # revealed: bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(...) + reveal_type(self.dispatch) + + # a call rebinds the receiver, which maps the signature the cyclic return type sits in + # revealed: bound method Self@use.dispatch(frame: Literal[1]) -> bound method Self@use.dispatch(frame) -> bound method Self@use.dispatch(...) + reveal_type(self.dispatch(1)) +``` + +## a self-referential bound method written into a diagnostic + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def dispatch(self, frame): + return self.dispatch + + def use(self) -> int: + return self.dispatch # error: [invalid-return-type] +``` + +## two methods that hand back each other's bound method + +each comparison of the cyclic return type against itself asks for a copy of one signature freshened +past the other, so the next round needs one nonce more than the last. no two rounds are ever equal, +which leaves nothing for a memo or a cycle guard to close on — the freshening is bounded instead, +and a pair that runs past the bound is refused rather than pursued. + +```toml +[environment] +python-version = "3.13" + +[analysis] +sound-types = true +``` + +```py +class Tracer: + def trace(self, event): + if event: + return self.exception(event) + return self.trace + + def exception(self, event): + return self.trace + + def use(self): + # revealed: bound method Self@use.exception(event) -> bound method Self@use.trace(event) -> (bound method Self@use.trace(event) -> Divergent) | (bound method Self@use.trace(...)) + reveal_type(self.exception) +``` + +## a tuple grown in a loop, widened while a cycle is being recovered + +`args` gains an element every time round the loop, so no round of the fixed-point iteration ever +repeats the one before it. cycle recovery gives the lengths up — nothing in the program says where +the growth stops — and keeps the element type, which the program really does determine. + +giving them up means unioning the element types together, and the ordinary union builder simplifies +its elements against one another. those simplifications are relation checks, and a relation check on +a type variable standing for an unannotated parameter answers what that parameter's bound is by +inferring the whole enclosing body — a query the cycle being recovered is already running. salsa +rejects a recovery function that acquires a cycle head of its own, so recovery builds this union +without the simplification. + +```py +def f(sequence, **kw): + args = (sequence,) + for k in kw: + args = args + (k,) + + reveal_type(args) # revealed: tuple[sequence@f | str, ...] +``` + +## an `assert` whose narrowing takes away the reason it narrowed + +what a body requires of an unannotated parameter is read off the body, and the body is then checked +against what that reading produced, so the two are settled by running them against each other until +they agree. + +`assert isinstance(proto, int) and proto <= 5` says `proto` has to be an `int`. once that is +`proto`'s bound, though, `isinstance(proto, int)` is statically true — and an arm of an `and` that +is always true says nothing about which branch this is, so it is dropped and its narrowing goes with +it. the round after that has nothing to say about `proto`, which puts the bound back where it +started and lets the round after that find the narrowing again. neither round repeats the one before +it. + +what the body requires is a fact about the body, so a requirement one round found is not taken away +by a round that cannot find it. + +the signature is read from inside another function: reading it from the module asks for it before +anything has been inferred, which is not the order that reaches the cycle at all. + +```py +def opcode(stack, proto): + len(stack) + assert isinstance(proto, int) and proto <= 5 + +def check(): + reveal_type(opcode) # revealed: def opcode(stack: some Sized, proto: some int) +``` + +## a float literal a loop recomputes from itself + +basedpython folds arithmetic on float literals, so `t` is a different literal on every pass of the +loop: `0.1`, then `0.2`, then `0.4`. The type of a name bound in a loop is the union of every value +that reaches it, and each round of the fixed-point iteration adds the value the round before it +produced, so no round ever repeats the one before it. + +The union builder already stands a group of literals down to their instance type once a union +defined in terms of itself holds more of them than the fixed point can afford — which is how a loop +counting `int` literals settles on `int`. Float and complex literals have no such group, so nothing +bounded them. + +```by +def run(): + t = 0.1 + while True: + t = t * 2 + reveal_type(t) # revealed: float +``` + +## a float literal a loop recomputes from itself through a generic call + +The same growth reaches the union a generic call's solve builds, which is assembled separately from +the one that approximates the loop. `min` hands back whatever its arguments have in common, so the +literal the last pass produced comes back out of the call and goes round again. + +```by +def run(): + t = 0.1 + while True: + t = min(0.1, t * 2) + reveal_type(t) # revealed: float +``` + +## a complex literal a loop recomputes from itself + +Complex literals fold the same way and are held the same way, so they need the same bound. + +```by +def run(): + t = 1j + while True: + t = t * 2 + reveal_type(t) # revealed: complex +``` + +## float literals a loop does not recompute + +Nothing is given up when the values a loop binds do not depend on the ones it bound before, however +many of them there are. + +```by +def run(flag: int): + for _ in range(3): + if flag == 0: + t = 0.1 + elif flag == 1: + t = 0.2 + elif flag == 2: + t = 0.3 + elif flag == 3: + t = 0.4 + elif flag == 4: + t = 0.5 + else: + t = 0.6 + reveal_type(t) # revealed: 0.1 | 0.2 | 0.3 | 0.4 | 0.5 | 0.6 +``` diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 04a0bd56c1..e99c2868f4 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -1905,9 +1905,14 @@ impl<'db> Type<'db> { let unioned = UnionType::from_elements_cycle_recovery(db, env, [previous, self]); unioned.collapse_tuple_lengths(db, env) } - .recursive_type_normalized_impl_with_cycle(db, env, cycle) + // the widening comes before the normalizer, never after it: the tuple it builds has an + // element unioned out of the members it replaced, and an element mentioning the cycle's own + // marker is exactly what the normalizer folds back onto that marker. widening last would + // hand back a type the normalizer never saw, the next round would normalize it, and the + // widened and marked forms would alternate without either being reached .without_growing_tuple_lengths(db, env, previous) .without_growing_self_nesting(db, env, previous, cycle) + .recursive_type_normalized_impl_with_cycle(db, env, cycle) } /// basedpython: `self`, with a class nested inside itself that the fixed-point iteration @@ -2107,7 +2112,9 @@ impl<'db> Type<'db> { let mut elements = Vec::new(); for element in self.union_elements(db) { match element.exact_tuple_instance_spec(db) { - Some(spec) => elements.push(spec.homogeneous_element_type(db, env)), + Some(spec) => { + elements.push(spec.homogeneous_element_type_in_cycle_recovery(db, env)); + } None => rest.push(element), } } @@ -2157,15 +2164,17 @@ impl<'db> Type<'db> { .and_then(|instance| instance.tuple_spec(db, env)) .filter(|spec| spec.as_fixed_length().is_some()) { - Some(spec) => element_types.push(spec.homogeneous_element_type(db, env)), + Some(spec) => { + element_types.push(spec.homogeneous_element_type_in_cycle_recovery(db, env)); + } None => kept.push(*element), } } - let element = UnionType::from_elements(db, env, element_types); + let element = UnionType::from_elements_cycle_recovery(db, env, element_types); kept.push(Type::tuple(Some( crate::types::tuple::TupleType::homogeneous(db, env, element), ))); - UnionType::from_elements(db, env, kept) + UnionType::from_elements_cycle_recovery(db, env, kept) } pub(crate) fn is_deeply_nested(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> bool { @@ -8770,28 +8779,23 @@ impl<'db> Type<'db> { deferred.re_evaluate(db, env, operands) } - Type::FunctionLiteral(function) => visitor.visit(db, self, type_mapping, || { + Type::FunctionLiteral(function) => { + let mapped = Type::FunctionLiteral(function.apply_type_mapping_impl( + db, + type_mapping, + tcx, + visitor, + )); match type_mapping { // Promote the types within the signature before promoting the signature to its // callable form. TypeMapping::Promote( PromotionMode::On, PromotionKind::Regular | PromotionKind::RegularKeepingLiterals, - ) => Type::FunctionLiteral(function.apply_type_mapping_impl( - db, - type_mapping, - tcx, - visitor, - )) - .promote_impl(db, visitor.env), - _ => Type::FunctionLiteral(function.apply_type_mapping_impl( - db, - type_mapping, - tcx, - visitor, - )), + ) => mapped.promote_impl(db, visitor.env), + _ => mapped, } - }), + } Type::BoundMethod(method) => Type::BoundMethod(BoundMethodType::new( db, diff --git a/crates/ty_python_semantic/src/types/constraints.rs b/crates/ty_python_semantic/src/types/constraints.rs index 50f7c6fc8a..9246a0b9ef 100644 --- a/crates/ty_python_semantic/src/types/constraints.rs +++ b/crates/ty_python_semantic/src/types/constraints.rs @@ -2079,7 +2079,7 @@ impl ConstraintId { /// /// Atomic types and bare typevars have constructor depth zero. The typevar depth is `0` if `ty` /// does not contain any typevars. -fn max_constructor_and_typevar_depth<'db>( +pub(crate) fn max_constructor_and_typevar_depth<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, ty: Type<'db>, diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 0a394696ee..e29bc46119 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -7220,9 +7220,16 @@ pub(super) fn refutable_unpacking_applies<'db>( // length is not worth complaining about when the contents were never stated either. // `Any` is a different matter: it is what someone writes to say the contents are // anything, and `list[Any]` states the length just as precisely as `list[int]` does + // + // the divergence marker is the same case at its strongest. a tuple grown out of its own + // value — `self.t = (a,)` where `a` came from unpacking `self.t` — is widened to + // `tuple[Divergent, ...]` by cycle recovery, because the iteration had no fixed point to + // reach. that variable length is the recovery's own artefact, not something the program + // says, so a `may not have exactly one element` here would be complaining about our + // having given up rather than about the code if value_tuple .variable_element_type(db) - .is_some_and(|element| element.is_unknown()) + .is_some_and(|element| element.is_unknown() || element.is_divergent()) { return false; } diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 533de9d5d3..9d62db58c7 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -204,6 +204,30 @@ impl<'db> DisplaySettings<'db> { } } + /// Begin displaying the signature of `function`, or `None` when doing so would recurse. + /// + /// A function's signature can name the function itself: an inferred return type of + /// `self.f` makes `f` return a callable over `f`. Rendering that nests forever, so every + /// site that writes a signature belonging to a `FunctionType` must go through here — the + /// exhausted result is a truncated `(...)`. The depth limit catches the case where the + /// nested function is an equal-but-distinct value, as it is once a signature has been + /// rebound to a receiver. + #[must_use] + fn enter_function(&self, function: FunctionType<'db>) -> Option { + const MAX_FUNCTION_TYPE_DISPLAY_DEPTH: usize = 4; + if self.visited_function_types.contains(&function) + || self.visited_function_types.len() >= MAX_FUNCTION_TYPE_DISPLAY_DEPTH + { + return None; + } + let mut visited = (*self.visited_function_types).clone(); + visited.insert(function); + Some(Self { + visited_function_types: Rc::new(visited), + ..self.clone() + }) + } + #[must_use] pub(crate) fn preserve_long_unions(self) -> Self { Self { @@ -1701,9 +1725,29 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { .display_with(db, self.env, self.settings.clone()) .fmt_detailed(f), Type::BoundMethod(bound_method) => { - let function = bound_method.function(db); - let self_ty = bound_method.self_instance(db); - let bound_signatures = bound_method.bound_signatures(db); + let function = bound_method.function(self.db); + let self_ty = bound_method.self_instance(self.db); + + let write_prefix = |f: &mut TypeWriter<'_, '_, 'db>| { + f.set_invalid_type_annotation(); + f.write_str("bound method ")?; + DisplayMaybeParenthesizedType { + ty: self_ty, + db: self.db, + env: self.env, + settings: self.settings.singleline(), + } + .fmt_detailed(f)?; + f.write_char('.')?; + f.with_type(self.ty).write_str(function.name(self.db)) + }; + + let Some(settings) = self.settings.enter_function(function) else { + write_prefix(f)?; + return f.write_str("(...)"); + }; + + let bound_signatures = bound_method.bound_signatures(self.db); match bound_signatures.overloads.as_slice() { [signature] => { @@ -1711,50 +1755,38 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { signature.should_hide_self_from_display(db, self.env); let type_parameters = DisplayOptionalGenericContext { generic_context: signature.generic_context.as_ref(), - db, + db: self.db, hide_unused_self, }; - f.set_invalid_type_annotation(); - f.write_str("bound method ")?; - DisplayMaybeParenthesizedType { - ty: self_ty, - db, - env: self.env, - settings: self.settings.singleline(), - } - .fmt_detailed(f)?; - f.write_char('.')?; - f.with_type(self.ty).write_str(function.name(db))?; + write_prefix(f)?; type_parameters.fmt_detailed(f)?; signature .display_with( self.db, - env, - self.settings - .disallow_signature_name() - .name_already_written(), + self.env, + settings.disallow_signature_name().name_already_written(), ) .fmt_detailed(f) } signatures => { // TODO: How to display overloads? - if !self.settings.multiline { + if !settings.multiline { // TODO: This should ideally have a TypeDetail but we actually // don't have a type for @overload (we just detect the decorator) f.write_str("Overload")?; f.write_char('[')?; } - let separator = if self.settings.multiline { "\n" } else { ", " }; + let separator = if settings.multiline { "\n" } else { ", " }; let mut join = f.join(separator); for signature in signatures { join.entry(&signature.display_with( - db, + self.db, self.env, - self.settings.clone(), + settings.clone(), )); } join.finish()?; - if !self.settings.multiline { + if !settings.multiline { f.write_str("]")?; } Ok(()) @@ -2511,25 +2543,14 @@ struct DisplayFunctionType<'env, 'db> { impl<'db> FmtDetailed<'db> for DisplayFunctionType<'_, 'db> { fn fmt_detailed(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result { - // Detect self-referential function types to prevent infinite recursion, - // and limit display depth for chains of different function types - // (e.g. multiple redefinitions with `TypeOf[foo]` return types). - const MAX_FUNCTION_TYPE_DISPLAY_DEPTH: usize = 4; let env = self.env; let db = self.db; - if self.settings.visited_function_types.contains(&self.ty) - || self.settings.visited_function_types.len() >= MAX_FUNCTION_TYPE_DISPLAY_DEPTH - { + let Some(settings) = self.settings.enter_function(self.ty) else { f.set_invalid_type_annotation(); f.write_str("def ")?; write!(f, "{}", self.ty.name(db))?; return f.write_str("(...)"); - } - - let mut settings = self.settings.clone(); - let mut visited = (*settings.visited_function_types).clone(); - visited.insert(self.ty); - settings.visited_function_types = Rc::new(visited); + }; let signature = self.ty.signature(db); diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index be2c5fee6d..cadf56e695 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -1996,12 +1996,37 @@ impl<'db> FunctionType<'db> { ) } + /// An inferred signature can mention the function it belongs to: `def f(self): return + /// self.f` gives `f` a return type that contains a bound method of `f`. Mapping that + /// signature reaches the same function again, so the descent has to be guarded here rather + /// than at each of the [`Type`] variants that hold a `FunctionType`. pub(crate) fn apply_type_mapping_impl<'a>( self, db: &'db dyn Db, type_mapping: &TypeMapping<'a, 'db>, tcx: TypeContext<'db>, visitor: &ApplyTypeMappingVisitor<'_, 'db>, + ) -> Self { + let mapped = visitor.visit(db, Type::FunctionLiteral(self), type_mapping, || { + Type::FunctionLiteral(self.apply_type_mapping_to_signatures( + db, + type_mapping, + tcx, + visitor, + )) + }); + let Type::FunctionLiteral(mapped) = mapped else { + return self; + }; + mapped + } + + fn apply_type_mapping_to_signatures<'a>( + self, + db: &'db dyn Db, + type_mapping: &TypeMapping<'a, 'db>, + tcx: TypeContext<'db>, + visitor: &ApplyTypeMappingVisitor<'_, 'db>, ) -> Self { // Returned-callable rescoping and type-alias specialization should not rebuild signatures from the // function literal; doing so can re-enter recursive `TypeOf` evaluation. diff --git a/crates/ty_python_semantic/src/types/infer.rs b/crates/ty_python_semantic/src/types/infer.rs index e3b8c6d2bd..ba47f76812 100644 --- a/crates/ty_python_semantic/src/types/infer.rs +++ b/crates/ty_python_semantic/src/types/infer.rs @@ -610,7 +610,7 @@ impl<'db> InferExpression<'db> { expression: Expression<'db>, tcx: TypeContext<'db>, ) -> InferExpression<'db> { - if tcx.annotation().is_some() { + if tcx.is_worth_interning(db, &ProgramEnvironment::from_scope(expression.scope(db))) { InferExpression::WithContext(ExpressionWithContext::new(db, expression, tcx)) } else { InferExpression::Bare(expression) @@ -645,7 +645,7 @@ pub(super) struct ScopeWithContext<'db> { impl<'db> InferScope<'db> { fn new(db: &'db dyn Db, scope: ScopeId<'db>, tcx: TypeContext<'db>) -> InferScope<'db> { - if tcx.annotation().is_some() { + if tcx.is_worth_interning(db, &ProgramEnvironment::from_scope(scope)) { InferScope::WithContext(ScopeWithContext::new(db, scope, tcx)) } else { InferScope::Bare(scope) @@ -729,6 +729,24 @@ impl<'db> TypeContext<'db> { self.target } + /// basedpython: whether this context earns a query key of its own + /// + /// the key an annotated inference runs under is *interned*, so a context that differs from the + /// one the last round used is a different query, carrying a `Type::divergent` initial value + /// named after it. a context holding a divergence marker closes that into a loop: the marker is + /// named after the query, the query is named by the key, and the key holds the marker — so a + /// cycle interns one more key every round and never reaches a fixed point, however settled the + /// types it is computing already are + /// + /// a marker is the cycle's stand-in for a type it has not reached yet, which is no guidance at + /// all, so the context is dropped and the expression is inferred bare. the next round drops the + /// same context and asks for the same bare key, unchanged + fn is_worth_interning(self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> bool { + self.annotation().is_some_and(|annotation| { + !any_over_type(db, env, annotation, false, |ty| ty.is_divergent()) + }) + } + /// basedpython: the expected type a bare name in this context resolves /// against — the annotation, or the result type of the call this expression /// is the callee of diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index 97e95e4a3c..e145b5a8aa 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -34,7 +34,7 @@ use crate::reachability::ReachabilityConstraintsExtension; use crate::types::ProgramEnvironment; use crate::types::call::CallArguments; use crate::types::callable::CallableType; -use crate::types::constraints::ConstraintSetBuilder; +use crate::types::constraints::{ConstraintSetBuilder, max_constructor_and_typevar_depth}; use crate::types::function::OverloadLiteral; use crate::types::narrow::{NarrowingConstraint, infer_narrowing_constraints}; use crate::types::protocol_class::InlineProtocolMember; @@ -64,7 +64,7 @@ use crate::types::{ cycle_initial = |_, id, _| Type::divergent(id), cycle_fn = |db, cycle, previous: &Type<'db>, value: Type<'db>, overload: OverloadLiteral<'db>| { let env = &ProgramEnvironment::from_file(overload.program_file(db)); - value.cycle_normalized(db, env, *previous, cycle) + divergence_bounded(db, env, value, cycle).cycle_normalized(db, env, *previous, cycle) }, heap_size = ruff_memory_usage::heap_size, )] @@ -91,6 +91,41 @@ pub(crate) fn inferred_return_type<'db>( ) } +/// The deepest a recovered return type may nest before the recursion that built it +/// is called what it is. +/// +/// A hand-written return type is a constructor or two deep — `list[int]`, +/// `dict[str, list[int]]`. Anything far past that inside a cycle was assembled one +/// layer per iteration rather than written by anybody. +const RETURN_TYPE_NESTING_LIMIT: u16 = 8; + +/// `value` with a return type that grows a constructor deeper every iteration +/// replaced by the divergence marker it already stands for. +/// +/// A body that returns a call taking the function itself — `def g(n): return map(g, n)` +/// — has no return type to reach: it is `map[map[…]]` without end. Ordinarily +/// [`Type::cycle_normalized`] folds such a type back onto the marker the cycle started +/// from, but the marker only survives while the type is *built*; passing through a +/// generic call's solve leaves a concrete type behind with nothing left to fold on, and +/// the fixed point recedes by one constructor per iteration forever. +/// +/// So bound the nesting rather than the iterations: past the bound the value is +/// replaced by the cycle head's own `Divergent`, which is what the marker-preserving +/// path would have produced, and the next iteration reproduces it unchanged. The bound +/// reads only the value and the cycle, so the query stays a function of its inputs. +fn divergence_bounded<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + value: Type<'db>, + cycle: &salsa::Cycle, +) -> Type<'db> { + let (constructor_depth, _) = max_constructor_and_typevar_depth(db, env, value); + if constructor_depth < RETURN_TYPE_NESTING_LIMIT { + return value; + } + cycle.head_ids().next().map_or(value, Type::divergent) +} + /// The return type `node`'s body determines, given what its expressions were inferred as. /// /// [`inferred_return_type`] reads those out of a completed scope inference. The @@ -314,9 +349,19 @@ pub(crate) fn inferred_parameter_bound<'db>( /// /// Every parameter is answered in one pass, because the expensive half — inferring /// the body, and re-binding each call in it — is shared between them. +/// +/// This reads the body, and the body is checked against the bounds this produces, so +/// the answer is reached by iterating the two to a fixed point. Each round is allowed +/// to say more about a parameter than the round before it, and to say something +/// different; the one thing it may not do is stop answering for a parameter it has +/// already answered for, which is what [`ParameterConstraints::keeping_requirements_seen`] +/// enforces. #[salsa::tracked( returns(ref), cycle_initial = |_, _, _| ParameterConstraints::default(), + cycle_fn = |_, _, previous: &ParameterConstraints<'db>, value: ParameterConstraints<'db>, _| { + value.keeping_requirements_seen(previous) + }, heap_size = ruff_memory_usage::heap_size, )] pub(crate) fn body_parameter_constraints<'db>( @@ -417,6 +462,43 @@ impl<'db> ParameterConstraints<'db> { .map(|(_, ty)| *ty) .collect() } + + /// This round's requirements, plus those of every parameter this round stopped + /// answering for. + /// + /// What a body requires of a parameter is a fact about the body, so a requirement one + /// round of the cycle found does not stop holding because a later round could not find + /// it. A round really can lose one. `assert isinstance(x, int) and x <= 5` narrows `x` + /// to `int` only while `isinstance(x, int)` can still come out false; the round after + /// that narrowing has become `x`'s bound the test is statically true, and an `and` arm + /// that is always true says nothing about which branch this is, so it is dropped — + /// taking its narrowing with it. That puts the bound back where it started, and the + /// round after finds the narrowing again. Neither round repeats the one before it, and + /// the iteration has no fixed point to reach. + /// + /// Requirements only ever being added is what leaves it one. + fn keeping_requirements_seen(mut self, previous: &Self) -> Self { + let dropped: Vec<_> = previous + .entries + .iter() + .filter(|(parameter, _)| { + !self + .entries + .iter() + .any(|(answered, _)| answered == parameter) + }) + .copied() + .collect(); + if dropped.is_empty() { + return self; + } + + let mut entries = self.entries.into_vec(); + entries.extend(dropped); + entries.sort_by_key(|(parameter, _)| *parameter); + self.entries = entries.into_boxed_slice(); + self + } } /// Feed each `assert` at the top level of the body back into the value it is about. diff --git a/crates/ty_python_semantic/src/types/relation.rs b/crates/ty_python_semantic/src/types/relation.rs index 17741030a1..8e9b5b4654 100644 --- a/crates/ty_python_semantic/src/types/relation.rs +++ b/crates/ty_python_semantic/src/types/relation.rs @@ -1251,6 +1251,17 @@ impl<'a, 'c, 'db> TypeRelationChecker<'a, 'c, 'db> { self.never() } + /// The answer for a pair of signatures whose comparison reproduces itself at an ever + /// greater typevar freshness. + /// + /// Such a pair is undecidable for the same reason a recursively-specialized structural + /// type is, and gets the same conservative answer — see + /// [`Self::recursive_type_pair_fallback`]. Rejecting is what keeps both members of a + /// self-referential union rather than folding one into the other. + pub(super) fn diverged_signature_pair(&self) -> ConstraintSet<'db, 'c> { + self.never() + } + /// Is `target` a metaclass instance (a nominal instance of a subclass of `builtins.type`)? /// /// This does not include all types that are subtypes of `builtins.type`! The semantic diff --git a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs index d80b97d412..e7572099c2 100644 --- a/crates/ty_python_semantic/src/types/set_theoretic/builder.rs +++ b/crates/ty_python_semantic/src/types/set_theoretic/builder.rs @@ -1025,7 +1025,11 @@ impl<'db> UnionBuilder<'db> { self.elements.swap_remove(index); } } - _ => self.push_type(ty, seen_aliases), + _ => { + if !self.widen_ungrouped_literals(literal, seen_aliases) { + self.push_type(ty, seen_aliases); + } + } } } // Adding `object` to a union results in `object`. @@ -1034,6 +1038,71 @@ impl<'db> UnionBuilder<'db> { } } + /// basedpython: stand a recursively defined union's float and complex literals down to + /// their instance type once there are more of them than the fixed point can afford. + /// + /// Every literal kind handled above lives in a [`UnionElement`] group of its own, and + /// widening that group is what stops a union defined in terms of itself from gaining an + /// element on every round of the fixed-point iteration. Float and complex literals — + /// which only basedpython has, and only basedpython does arithmetic on — are kept as + /// ordinary union elements, so nothing bounded them. A loop that computes the next value + /// from the last, `t = t * 2`, folds `0.1 | 0.2 | 0.4 | …` one element longer each round, + /// so no round repeats the one before it and the iteration never converges. + /// + /// The one type a loop like that really determines is `float`, so past the limit that is + /// what the literals stand for. Both places a growing union is assembled are covered: the + /// loop-header union that approximates a loop's fixed point, and the union a cycle + /// recovery function rebuilds. + /// + /// The rule is written for every literal kind without a group rather than for these two, + /// so a kind added later is bounded whether or not anybody remembers this. The kinds that + /// cannot proliferate — a `bool`, a `LiteralString` — never reach the limit anyway. + /// + /// Returns `true` when the literal is already accounted for and the caller must not add + /// it. + fn widen_ungrouped_literals( + &mut self, + literal: LiteralValueType<'db>, + seen_aliases: &mut Vec>, + ) -> bool { + // A union not defined in terms of itself is complete as soon as it is built, so there + // is nothing to converge and no reason to give its literals up. + if !self.recursively_defined.is_yes() { + return false; + } + + let db = self.db; + let fallback = literal.fallback_instance(db, &self.env); + let mut same_kind = SmallVec::<[usize; 8]>::new(); + for (index, element) in self.elements.iter().enumerate() { + let UnionElement::Type(existing) = element else { + continue; + }; + // Once widened, the instance type stands for every literal of its kind, this one + // included. Outside recovery `push_type` reaches the same conclusion through the + // ordinary redundancy check, which also applies the simplifications skipped here. + if self.cycle_recovery && *existing == fallback { + return true; + } + if let Type::LiteralValue(existing_literal) = existing + && existing_literal.fallback_instance(db, &self.env) == fallback + { + same_kind.push(index); + } + } + + if same_kind.len() < MAX_RECURSIVE_UNION_LITERALS { + return false; + } + + // Removing from the back leaves the earlier indices where they were. + for index in same_kind.into_iter().rev() { + self.elements.remove(index); + } + self.add_in_place_impl(fallback, seen_aliases); + true + } + fn push_type(&mut self, ty: Type<'db>, seen_aliases: &mut Vec>) { let env = &self.env; let db = self.db; @@ -2084,6 +2153,60 @@ mod tests { assert_eq!(union.build(), KnownClass::Int.to_instance(db, &env)); + // basedpython: float literals have no `UnionElement` group of their own, so they are + // bounded by `widen_ungrouped_literals` instead. A loop doubling a float — the shape + // that first ran the fixed point out of iterations — reaches the limit this way. One + // value past the limit, since it takes that many to widen. + let doubling = || { + (0..=MAX_RECURSIVE_UNION_LITERALS).scan(0.1, |value, _| { + let doubled = *value; + *value *= 2.0; + Some(doubled) + }) + }; + let over_limit = doubling().map(Type::float_literal).collect::>(); + + let float_union = over_limit.iter().copied().fold( + UnionBuilder::new(db, &env) + .cycle_recovery(true) + .recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!(float_union.build(), KnownClass::Float.to_instance(db, &env)); + + // The loop-header union that approximates a loop's fixed point is not built in + // recovery mode, and it grows the same way. + let float_loop_union = over_limit.iter().copied().fold( + UnionBuilder::new(db, &env).recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!( + float_loop_union.build(), + KnownClass::Float.to_instance(db, &env) + ); + + let complex_union = doubling() + .map(|value| Type::complex_literal(db, 0.0, value)) + .fold( + UnionBuilder::new(db, &env) + .cycle_recovery(true) + .recursively_defined(RecursivelyDefined::Yes), + UnionBuilder::add, + ); + assert_eq!( + complex_union.build(), + KnownClass::Complex.to_instance(db, &env) + ); + + // A union nothing defines in terms of itself settles in one go, so its literals are + // kept however many there are. + let non_recursive = over_limit + .iter() + .copied() + .fold(UnionBuilder::new(db, &env), UnionBuilder::add) + .build(); + assert_ne!(non_recursive, KnownClass::Float.to_instance(db, &env)); + let assert_widens = |literal, instance| { for (first, second) in [(literal, instance), (instance, literal)] { let union = UnionBuilder::new(db, &env) diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 0411d8a1c8..509b3ab6ae 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -42,7 +42,8 @@ use crate::types::relation::{ use crate::types::tuple::{Tuple, TupleType, VariableSegment}; use crate::types::typed_dict::extract_unpacked_typed_dict_keys_from_kwargs_annotation; use crate::types::typevar::{ - TypeVarInstance, TypeVarKind, TypeVarSet, max_typevar_freshness_matching_generic_context, + MAX_TYPEVAR_FRESHNESS_DELTA, TypeVarInstance, TypeVarKind, TypeVarSet, + max_typevar_freshness_matching_generic_context, }; use crate::types::{ ApplyTypeMappingVisitor, BindingContext, BoundTypeVarIdentity, BoundTypeVarInstance, @@ -2683,6 +2684,9 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { .max_typevar_freshness_matching_generic_context(db, generic_context) .map(|freshness| freshness.increment().value()) { + if delta > MAX_TYPEVAR_FRESHNESS_DELTA { + return self.diverged_signature_pair(); + } freshened_source = source.freshen_bound_typevars(db, env, delta); &freshened_source } else { @@ -2695,6 +2699,9 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { .max_typevar_freshness_matching_generic_context(db, generic_context) .map(|freshness| freshness.increment().value()) { + if delta > MAX_TYPEVAR_FRESHNESS_DELTA { + return self.diverged_signature_pair(); + } freshened_target = target.freshen_bound_typevars(db, env, delta); &freshened_target } else { diff --git a/crates/ty_python_semantic/src/types/tuple.rs b/crates/ty_python_semantic/src/types/tuple.rs index 3a9714f301..150fee371f 100644 --- a/crates/ty_python_semantic/src/types/tuple.rs +++ b/crates/ty_python_semantic/src/types/tuple.rs @@ -2410,14 +2410,49 @@ impl<'db> Tuple, VariableSegment<'db>> { db: &'db dyn Db, env: &ProgramEnvironment<'db>, ) -> Type<'db> { + self.element_types_unioned(db, env, false) + } + + /// basedpython: the same answer as [`Self::homogeneous_element_type`], for a caller that is + /// itself part of a cycle recovery function. + /// + /// The ordinary union builder simplifies its elements against one another, and a relation + /// check can force arbitrary inference — a type variable standing for an unannotated + /// parameter answers what its bound is by inferring the whole enclosing function body. A + /// recovery function that reached a query already on the stack would acquire a cycle head of + /// its own, which salsa rejects outright, so recovery builds its unions without that + /// simplification. + pub(crate) fn homogeneous_element_type_in_cycle_recovery( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Type<'db> { + self.element_types_unioned(db, env, true) + } + + fn element_types_unioned( + &self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + cycle_recovery: bool, + ) -> Type<'db> { + // cycle recovery mode already leaves aliases packed, so the two modes agree about aliases + let mut builder = UnionBuilder::new(db, env) + .unpack_aliases(false) + .cycle_recovery(cycle_recovery); match self { Tuple::Fixed(tuple) => { - UnionType::from_elements_leave_aliases(db, env, tuple.iter_all_elements()) + for element in tuple.iter_all_elements() { + builder.add_in_place(element); + } } Tuple::Variable(tuple) => { - UnionType::from_elements_leave_aliases(db, env, tuple.iter_all_elements(db)) + for element in tuple.iter_all_elements(db) { + builder.add_in_place(element); + } } } + builder.build() } fn tuple_class_type(&self, db: &'db dyn Db, env: &ProgramEnvironment<'db>) -> Type<'db> { diff --git a/crates/ty_python_semantic/src/types/typevar.rs b/crates/ty_python_semantic/src/types/typevar.rs index 96c128a8ad..fab5cb0f91 100644 --- a/crates/ty_python_semantic/src/types/typevar.rs +++ b/crates/ty_python_semantic/src/types/typevar.rs @@ -1044,6 +1044,16 @@ pub struct TypeVarNonce(u32); // This type does not have any heap storage. impl get_size2::GetSize for TypeVarNonce {} +/// How far a signature's typevars may be freshened past the signature it is compared against. +/// +/// Freshening only has to lift one signature's typevars clear of the other's, so the distance +/// needed is the nesting of same-context generic signatures inside one comparison — a handful +/// at most in real code. It is unbounded only when a comparison reproduces itself at a greater +/// freshness, which a self-referential inferred return type does: `def f(self): return self.f` +/// makes each comparison of `f` against itself demand a signature one nonce fresher than the +/// last, so no two rounds are ever equal and no memo or cycle guard can close the loop. +pub(crate) const MAX_TYPEVAR_FRESHNESS_DELTA: u32 = 32; + impl TypeVarNonce { pub(crate) const NONE: Self = Self(0); const FIRST: Self = Self(1); From af68fc0508e340bca0e18511126ba0d542848cd2 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:35:07 +1000 Subject: [PATCH 2/7] a command works on the files it was given, and a negated exclude means what it says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `by compile a.py` computed its file list from the arguments and then used it only for an emptiness check, compiling every source in the project instead. that is not a harmless superset: it costs every other module's build time, it fails for a diagnostic in a file nobody named, and it silently compiles whatever sits beside the file under test — which invalidated a delta-debugging run whose original was in the same directory as each candidate. the database still holds the whole project, because a type imported from a sibling has to resolve; only the checking and the emission are narrowed. two exclude bugs, found while establishing that the third — the one that was reported — does not exist: `exclude = ["!dist"]` works and always did, and only the forms gitignore itself refuses (`!dist/**`, inside a directory the walk never descends into) do not. a rooted negation lost its `!` in `into_absolute` and registered as a positive exclude, the exact opposite of what was written, with no diagnostic. and an unreachable negation was a silent no-op; it now reports `unreachable-exclude-negation` and names the spelling that would work. `by_commands.rs` also stopped applying its own hard-coded directory list after the project filter, where it had been re-dropping files a negation deliberately re-included. --- crates/ty/docs/configuration.md | 8 +- crates/ty/src/by_commands.rs | 236 ++++++++++++++++++++-- crates/ty/tests/by_e2e.rs | 173 ++++++++++++++++ crates/ty/tests/cli/file_selection.rs | 99 +++++++++ crates/ty_project/src/glob/exclude.rs | 85 ++++++++ crates/ty_project/src/glob/portable.rs | 106 +++++++++- crates/ty_project/src/metadata/options.rs | 92 ++++++++- ty.schema.json | 2 +- 8 files changed, 777 insertions(+), 24 deletions(-) diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index bed525a5f2..31bcbdeb43 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -2073,7 +2073,13 @@ By default, ty excludes commonly ignored directories: - `**/venv/` You can override any default exclude by using a negated pattern. For example, -to re-include `dist` use `exclude = ["!dist"]` +to re-include `dist` use `exclude = ["!dist"]`, or `exclude = ["!**/dist/"]` to +re-include every `dist` directory rather than only the one at the project root. + +A negated pattern can only re-include something that is still walked, so it cannot +reach into a directory that is itself excluded. `exclude = ["!dist/generated.py"]` +re-includes nothing, because the walk stops at `dist`. Re-include the directory +first: `exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"]` **Default value**: `null` diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 328478b5c5..62ed8e698e 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::ffi::OsStr; use std::fs; use std::io::{self, Read}; @@ -222,21 +223,30 @@ pub(crate) fn cmd_run( for entry in &traceback_entries { let source = fs::read_to_string(&entry.by_path) .with_context(|| format!("could not read {}", entry.by_path.display()))?; - let name = entry + // the generated tree *is* the module tree, so the dotted name is the + // path within it — and it has to be dotted, because a class's + // `__module__` is read off the front of its type's `tp_name`. a file + // the tree gives no name to is left interpreted rather than compiled + // under a guessed one + let relative = entry .py_path - .file_stem() - .and_then(|stem| stem.to_str()) - .context("a source file has no usable module name")?; - if name == module { + .strip_prefix(tmp.path()) + .unwrap_or(&entry.py_path); + let Some(name) = dotted_module_name(relative) else { + continue; + }; + if name.dotted() == module { continue; } - let dir = entry.py_path.parent().unwrap_or(tmp.path()); let mut lowered = by_irbuild::module_from_source(&source, name, options.language); lowered.lines = Some(by_ir::function::LineTable::new( entry.by_path.display().to_string(), &source, )); - by_build::build_lowered(lowered, &source, &toolchain, dir, &options) + // the root of the generated tree, not the directory the `.py` landed + // in: the build lays the extension out at its module's own place, and + // handing it the leaf directory would nest the tree inside itself + by_build::build_lowered(lowered, &source, &toolchain, tmp.path(), &options) .with_context(|| format!("could not compile {}", entry.by_path.display()))?; built += 1; } @@ -310,6 +320,89 @@ fn module_relative_path(roots: &[PathBuf], root: &Path, bpy: &Path) -> PathBuf { .with_extension("py") } +/// The dotted module name a file laid out at `relative` will be imported under. +/// +/// The tree the generated python is written into *is* the module tree — every +/// file lands at [`module_relative_path`] — so the name is that path with its +/// separators turned into dots. `pkg/__init__.py` is the package `pkg` itself, +/// which is the name a class defined in it reports as its `__module__`. +/// +/// `None` when the path names no module: nothing but `__init__.py` at the root, +/// or a component that is not plain text. A caller with no name has nothing to +/// compile, and guessing one would be worse than saying so. +fn dotted_module_name(relative: &Path) -> Option { + let mut components: Vec<&str> = Vec::new(); + for component in relative.components() { + let std::path::Component::Normal(name) = component else { + return None; + }; + components.push(name.to_str()?); + } + let last = components.pop()?; + let stem = last.strip_suffix(".py").unwrap_or(last); + // a package's `__init__` is not a module beside the package, it *is* the + // package — so a class written in it belongs to the package's own name + let is_package = stem == "__init__"; + if !is_package { + components.push(stem); + } + if components.is_empty() { + return None; + } + let dotted = components.join("."); + Some(if is_package { + by_ir::ModuleName::package(dotted) + } else { + by_ir::ModuleName::new(dotted) + }) +} + +/// The dotted module name `file` is imported under, as the project resolves it. +/// +/// The resolver is what knows this, and nothing simpler will do: it accounts for +/// the search paths (`src/pkg/m.py` in a src-layout project is `pkg.m`, not +/// `src.pkg.m`), for a `pkg/__init__.py` whose module is `pkg`, and for a +/// namespace package, which a walk looking for `__init__.py` would stop short of. +/// It also refuses a directory that merely holds `.py` files — cpython's own +/// `config-3.13-darwin` is not a package, and its name is not even an identifier. +/// +/// `None` when no search path reaches the file. Such a file has no dotted name to +/// be had: the only way to import it is from its own directory, under its stem. +fn resolved_module_name(db: &ProjectDatabase, file: ruff_db::files::File) -> Option { + let program_file = ty_python_semantic::Db::program_file(db, file); + let module = ty_module_resolver::file_to_module(db, program_file.resolver_file(db))?; + Some(module.name(db).to_string()) +} + +/// The name `path` is compiled under — dotted, and knowing whether it is a +/// package's. +/// +/// The *dotted* name because it is what the emitted types carry: cpython reads a +/// class's `__module__` off the front of its `tp_name`, so a class in +/// `tkinter/m.py` compiled as plain `m` reports a module nothing can look up — +/// and `dataclasses` does exactly that lookup. +/// +/// Package or not because the two are written to different files. The resolver +/// names a package after its directory, so an `__init__.py` it resolved is that +/// package, and its artefact is the `__init__` inside the directory. A file the +/// resolver could not reach has no dotted name at all: it falls back to its stem, +/// which is the only name it could be imported under, from its own directory. +fn compiled_module_name( + db: &ProjectDatabase, + path: &Path, + file: ruff_db::files::File, +) -> anyhow::Result { + let stem = path + .file_stem() + .and_then(|stem| stem.to_str()) + .context("a source file has no usable module name")?; + Ok(match resolved_module_name(db, file) { + Some(resolved) if stem == "__init__" => by_ir::ModuleName::package(resolved), + Some(resolved) => by_ir::ModuleName::new(resolved), + None => by_ir::ModuleName::new(stem), + }) +} + /// The project's `run.main` entry point, if one is configured. fn configured_main(db: &ProjectDatabase) -> Option { let options = db.project().metadata(db).options(); @@ -435,7 +528,40 @@ pub(crate) fn cmd_compile( // gradual, and `--no-any` would then fail on noise // `compile` embeds fallback source produced by the untyped transpile, which // takes no db, so the rebuilder the other commands thread through is unused here - let (db, handles, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + let (db, project, _rebuilder, _root) = build_project_db(&cwd, COMPILABLE_SOURCES)?; + + // the database holds the whole project so a type imported from a sibling + // resolves, but only the files that were *asked for* are checked and emitted. + // compiling the project regardless of the arguments is not a harmless + // superset: it costs every other module's build time, it fails the command + // for a diagnostic in a file nobody named, and — because the argument is + // ignored rather than rejected — it silently compiles a file beside the one + // under test, which has already invalidated one delta-debugging run here + let requested: Vec = sources + .iter() + .map(|source| { + if source.is_absolute() { + source.clone() + } else { + cwd.join(source) + } + }) + .map(|source| source.canonicalize().unwrap_or(source)) + .collect(); + let handles: Vec<_> = project + .iter() + .filter(|(path, _)| { + files.is_empty() + || requested.iter().any(|wanted| { + *wanted == **path || path.canonicalize().is_ok_and(|p| p == *wanted) + }) + }) + .cloned() + .collect(); + if handles.is_empty() { + eprintln!("no .by or .py files found"); + return Ok(ExitStatus::Success); + } // a source with nothing to lower blocks, the way it does for `build` and // `transpile` — it could not be parsed, or could not be read at all. a *type* @@ -459,13 +585,32 @@ pub(crate) fn cmd_compile( render_diagnostics(&db, &diagnostics)?; } + // what each source will be compiled as, worked out before anything is written: + // two sources that land on the same artefact used to leave only the second, and + // nothing said so + let mut names: Vec = Vec::with_capacity(handles.len()); + let mut claimed: HashMap = HashMap::new(); for (path, file) in &handles { + let name = compiled_module_name(&db, path, *file)?; + // keyed on the artefact rather than on the name, because the artefact is + // what would be overwritten — and a file the resolver cannot name falls + // back to its stem, which two directories can share + let artifact = name.relative_path(""); + if let Some(first) = claimed.insert(artifact, path.as_path()) { + anyhow::bail!( + "`{}` and `{}` would both be compiled as the module `{}`, \ + and the second would replace the first's artifact", + first.display(), + path.display(), + name.dotted() + ); + } + names.push(name); + } + + for ((path, file), name) in handles.iter().zip(names) { let source = fs::read_to_string(path) .with_context(|| format!("could not read {}", path.display()))?; - let module = path - .file_stem() - .and_then(|stem| stem.to_str()) - .context("a source file has no usable module name")?; let program_file = ty_python_semantic::Db::program_file(&db, *file); let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); @@ -480,7 +625,7 @@ pub(crate) fn cmd_compile( &model.program_environment(), &model, parsed.suite(), - module, + name, options.language.unique_loop_bindings(), ); // the real path, so a `#line` in the generated C resolves for a debugger @@ -1045,6 +1190,29 @@ const BY_SOURCES: &[&str] = &["by", "byi"]; /// output directory rather than beside the source, so a `.py` is an input const COMPILABLE_SOURCES: &[&str] = &["by", "byi", "py"]; +/// The [`NON_SOURCE_DIRS`] entries that ty's own `src.exclude` defaults don't already drop. +/// +/// [`is_hidden_within`] runs over files that have *already* passed the project's file filter, +/// so for a name ty excludes by default — `venv`, `dist`, `node_modules`, `.tox`, … — a file +/// can only have reached it because the configuration deliberately re-included the directory +/// with a negated pattern, which `src.exclude` documents as the way to override a default. +/// Re-dropping such a file here would quietly undo that, and it's why a project could not +/// compile a module of its own that happens to live in a directory named `venv`. +/// +/// What's left are the names ty has no default opinion about, where this walk is the only +/// thing keeping a dependency tree or a build output out of the emitted set. The unfiltered +/// [`NON_SOURCE_DIRS`] still applies to [`may_contain_sources`], which walks the file system +/// directly and never sees the project configuration at all. +const NON_SOURCE_DIRS_TY_ALLOWS: &[&str] = &[ + "env", + ".env", + "site-packages", + "__pycache__", + ".pytest_cache", + "build", + "out", +]; + /// Whether `path` sits inside a hidden or build-output directory under `root`. fn is_hidden_within(path: &Path, root: &Path) -> bool { let Ok(relative) = path.strip_prefix(root) else { @@ -1055,7 +1223,7 @@ fn is_hidden_within(path: &Path, root: &Path) -> bool { .into_iter() .flat_map(Path::components) .filter_map(|component| component.as_os_str().to_str()) - .any(|name| name.starts_with('.') || NON_SOURCE_DIRS.contains(&name)) + .any(|name| name.starts_with('.') || NON_SOURCE_DIRS_TY_ALLOWS.contains(&name)) } /// Build a project db rooted at `cwd`, returning it alongside the @@ -1287,7 +1455,10 @@ pub(crate) fn cmd_version_by(output_format: crate::args::HelpFormat) -> ExitStat #[cfg(test)] mod tests { - use super::{is_hidden_within, module_relative_path, reverse_dir, reverse_dir_converting}; + use super::{ + dotted_module_name, is_hidden_within, module_relative_path, reverse_dir, + reverse_dir_converting, + }; use crate::ExitStatus; use by_transforms::config::Config; use std::path::{Path, PathBuf}; @@ -1313,6 +1484,37 @@ mod tests { ); } + /// a compiled module carries its name into every type it emits, and cpython + /// reads a class's `__module__` off the front of that — so the name of a file + /// inside the tree is the whole path to it, not the file's own stem + #[test] + fn a_file_inside_the_tree_is_named_for_its_whole_path() { + assert_eq!( + dotted_module_name(Path::new("pkg/sub/main.py")), + Some(by_ir::ModuleName::new("pkg.sub.main")) + ); + assert_eq!( + dotted_module_name(Path::new("top.py")), + Some(by_ir::ModuleName::new("top")) + ); + } + + /// a package's `__init__` is not a module beside the package, it *is* the + /// package — which is the module a class written in it belongs to + #[test] + fn a_packages_init_is_named_for_the_package() { + let name = dotted_module_name(Path::new("pkg/__init__.py")); + assert_eq!(name, Some(by_ir::ModuleName::package("pkg"))); + // and its artefact is the `__init__` inside the package, not a file + // called `pkg` beside it — that is the only place cpython's finder looks + assert_eq!( + name.map(|name| name.relative_path(".so")), + Some(PathBuf::from("pkg/__init__.so")) + ); + // and at the root there is no package for it to be, so there is no name + assert_eq!(dotted_module_name(Path::new("__init__.py")), None); + } + /// a root that shares no prefix with the file — which is what `canonicalize` /// and `current_dir` disagreeing produced on windows — must not leave the /// path absolute: joining that onto the output directory discards the output @@ -1340,6 +1542,10 @@ mod tests { assert!(!is_hidden_within(Path::new("/p/src/pkg/main.by"), root)); // the file's own name is not a directory component assert!(!is_hidden_within(Path::new("/p/.hidden.by"), root)); + // a name ty excludes by default is left to the project filter, so that a + // negated `src.exclude` pattern re-including it isn't quietly undone here + assert!(!is_hidden_within(Path::new("/p/venv/__init__.by"), root)); + assert!(!is_hidden_within(Path::new("/p/dist/main.by"), root)); } /// a source that declares its own encoding converts like any other, and the diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 3ba786aa0d..8c9260df37 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -48,6 +48,179 @@ fn run_transpile(source: &str, extra_args: &[&str]) -> String { String::from_utf8(output.stdout).unwrap() } +#[test] +fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { + // `by compile a.py` used to compile every source in the project and ignore the + // argument entirely. that is not a harmless superset: it costs every other + // module's build time, it fails the command for a diagnostic in a file nobody + // named, and it silently compiles a file sitting beside the one under test — + // which invalidated a delta-debugging run whose original was in the same + // directory as each candidate + // + // the database still holds the whole project, because a type imported from a + // sibling has to resolve. that is what `lib.py` is here to prove: it is never + // compiled, and `wanted.py` still lowers `Point` rather than declining + let dir = std::env::temp_dir().join("by_cli_only_named"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + std::fs::write( + dir.join("lib.py"), + "class Point:\n def __init__(self) -> None:\n self.x: int = 7\n", + ) + .unwrap(); + std::fs::write( + dir.join("wanted.py"), + "from lib import Point\n\n\ndef go() -> int:\n p = Point()\n return p.x\n", + ) + .unwrap(); + std::fs::write( + dir.join("other.py"), + "def unrelated() -> int:\n return 2\n", + ) + .unwrap(); + + let out = dir.join("out"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "wanted.py", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + + assert!(out.join("wanted.c").exists(), "the named file is compiled"); + assert!( + !out.join("lib.c").exists() && !out.join("other.c").exists(), + "a file that was not named is not compiled" + ); + + // the cross-module type resolved: a declined body would not carry the + // attribute read at all + let emitted = std::fs::read_to_string(out.join("wanted.c")).expect("the C is readable"); + assert!( + emitted.contains("by_wanted_go"), + "`go` lowered natively, so `Point` resolved out of the uncompiled sibling" + ); +} + +/// write a project of package members under `dir`, each answering with its own +/// dotted name +fn write_package_project(dir: &std::path::Path) { + let _ = std::fs::remove_dir_all(dir); + std::fs::create_dir_all(dir.join("pkg/sub")).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + for (path, tag) in [ + ("pkg/__init__.py", 1), + ("pkg/sub/__init__.py", 2), + ("pkg/dup.py", 3), + ("pkg/sub/dup.py", 4), + ] { + std::fs::write( + dir.join(path), + format!("def tag() -> int:\n return {tag}\n"), + ) + .unwrap(); + } +} + +#[test] +fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { + // `by compile -o out` used to write every artefact flat, named after the + // module's last component. two members of a package sharing a last component + // then wrote the same file and the second silently won — and *no* package + // member's artefact was importable under the name it had been compiled as, + // because a flat `dup.so` can only ever be imported as `dup` + let dir = std::env::temp_dir().join("by_cli_package_tree"); + write_package_project(&dir); + + let out = dir.join("o"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!( + result.status.success(), + "by exited with error:\n{}", + String::from_utf8_lossy(&result.stderr) + ); + + // four sources, four artefacts — a package's own file is the `__init__` inside + // its directory, which is the only name cpython's finder looks for + for relative in [ + "pkg/__init__.c", + "pkg/sub/__init__.c", + "pkg/dup.c", + "pkg/sub/dup.c", + ] { + assert!(out.join(relative).exists(), "{relative} was written"); + } + // and nothing named after a last component alone + assert!( + !out.join("dup.c").exists() && !out.join("sub.c").exists() && !out.join("pkg.c").exists() + ); + + // the two `dup` members are distinct modules, not one file written twice + let first = fs::read_to_string(out.join("pkg/dup.c")).unwrap(); + let second = fs::read_to_string(out.join("pkg/sub/dup.c")).unwrap(); + assert!(first.contains("by_pkg_dup_tag"), "{first}"); + assert!(second.contains("by_pkg_sub_dup_tag"), "{second}"); +} + +#[test] +fn compile_refuses_two_sources_that_would_write_the_same_artifact() { + // laying the output out as the module tree settles the collision between two + // package members, but not this one: neither directory here has a name python + // could import, so neither file has a dotted name and both fall back to their + // stem. one artefact would be written twice and only the second kept, which is + // the silent loss the tree was meant to end — so it is refused before anything + // is written rather than half-performed + let dir = std::env::temp_dir().join("by_cli_artifact_clash"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(dir.join("a-one")).unwrap(); + std::fs::create_dir_all(dir.join("b-two")).unwrap(); + std::fs::write( + dir.join("pyproject.toml"), + "[project]\nname=\"s\"\nversion=\"0\"\nrequires-python=\">=3.13\"\n", + ) + .unwrap(); + std::fs::write(dir.join("a-one/m.py"), "def tag() -> int:\n return 1\n").unwrap(); + std::fs::write(dir.join("b-two/m.py"), "def tag() -> int:\n return 2\n").unwrap(); + + let out = dir.join("o"); + let result = Command::new(env!("CARGO_BIN_EXE_by")) + .args(["compile", "-o"]) + .arg(&out) + .arg("--emit-c-only") + .current_dir(&dir) + .output() + .expect("failed to spawn by"); + assert!(!result.status.success(), "the clash is refused"); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!( + stderr.contains("would both be compiled as the module `m`"), + "{stderr}" + ); + // and it said so before writing either one + assert!(!out.join("m.c").exists(), "{stderr}"); +} + #[test] fn compile_transpiles_the_fallback_with_the_lowering_options_it_was_given() { // a declined function *runs* from the embedded source, so `by compile` has to diff --git a/crates/ty/tests/cli/file_selection.rs b/crates/ty/tests/cli/file_selection.rs index 841e259434..a38fe1f61c 100644 --- a/crates/ty/tests/cli/file_selection.rs +++ b/crates/ty/tests/cli/file_selection.rs @@ -544,6 +544,105 @@ fn remove_default_exclude() -> anyhow::Result<()> { Ok(()) } +/// A negation that names the directory re-includes it without needing the `**/` the default +/// exclude is written with, which is the spelling the `exclude` documentation gives. +#[test] +fn remove_default_exclude_unqualified() -> anyhow::Result<()> { + let case = CliTest::with_files([( + "dist/generated.py", + r#" + print(another_undefined_var) # error: unresolved-reference + "#, + )])?; + + for pattern in ["!dist", "!dist/", "!./dist"] { + case.write_file("ty.toml", &format!("[src]\nexclude = [\"{pattern}\"]\n"))?; + + insta::allow_duplicates! { + assert_cmd_snapshot!(case.command(), @" + success: false + exit_code: 1 + ----- stdout ----- + error[unresolved-reference]: Name `another_undefined_var` used when not defined + --> dist/generated.py:2:7 + | + 2 | print(another_undefined_var) # error: unresolved-reference + | ^^^^^^^^^^^^^^^^^^^^^ + + Found 1 diagnostic + + ----- stderr ----- + "); + } + } + + Ok(()) +} + +/// A negation can only re-include something the walk still reaches. Pointing one into a +/// directory that is itself excluded matches nothing at all, so it's reported rather than +/// silently doing nothing. +#[test] +fn negation_into_an_excluded_directory_warns() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ("src/main.py", "print('ok')"), + ("dist/generated.py", "print(dist_var)"), + ("dist/keep.py", "print(keep_var)"), + ])?; + + case.write_file( + "ty.toml", + r#" + [src] + exclude = ["!dist/keep.py"] + "#, + )?; + + assert_cmd_snapshot!(case.command(), @r#" + success: false + exit_code: 1 + ----- stdout ----- + warning[unreachable-exclude-negation]: Negated pattern `!dist/keep.py` has no effect + --> ty.toml:3:12 + | + 3 | exclude = ["!dist/keep.py"] + | ^^^^^^^^^^^^^^^ This pattern can never match + info: `dist` is excluded, so nothing inside it is ever reached + info: Re-include the directory first by adding `!**/dist/` + + Found 1 diagnostic + + ----- stderr ----- + "#); + + // Re-including the directory first makes the same negation reachable, so the warning goes + // away and only `keep.py` comes back. + case.write_file( + "ty.toml", + r#" + [src] + exclude = ["!**/dist/", "**/dist/**", "!dist/keep.py"] + "#, + )?; + + assert_cmd_snapshot!(case.command(), @" + success: false + exit_code: 1 + ----- stdout ----- + error[unresolved-reference]: Name `keep_var` used when not defined + --> dist/keep.py:1:7 + | + 1 | print(keep_var) + | ^^^^^^^^ + + Found 1 diagnostic + + ----- stderr ----- + "); + + Ok(()) +} + /// Test that configuration excludes can be removed via CLI negation #[test] fn cli_removes_config_exclude() -> anyhow::Result<()> { diff --git a/crates/ty_project/src/glob/exclude.rs b/crates/ty_project/src/glob/exclude.rs index c7f6e625b9..8d8a63ce6c 100644 --- a/crates/ty_project/src/glob/exclude.rs +++ b/crates/ty_project/src/glob/exclude.rs @@ -291,3 +291,88 @@ impl GitignoreBuilder { Ok(self) } } + +#[cfg(test)] +mod tests { + use ruff_db::system::SystemPath; + + use crate::GlobFilterCheckMode; + use crate::glob::exclude::{ExcludeFilter, ExcludeFilterBuilder}; + use crate::glob::{PortableGlobKind, PortableGlobPattern}; + use crate::metadata::options::DEFAULT_SRC_EXCLUDES; + + /// Builds an exclude filter the same way `build_exclude_filter` does: the defaults first, + /// anchored at the file system root, then the user's patterns anchored at the project root. + fn filter_with_defaults( + user_patterns: impl IntoIterator, + ) -> ExcludeFilter { + let mut builder = ExcludeFilterBuilder::new(); + + for pattern in DEFAULT_SRC_EXCLUDES { + builder + .add( + &PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute(""), + ) + .unwrap(); + } + + for pattern in user_patterns { + builder + .add( + &PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute("/project"), + ) + .unwrap(); + } + + builder.build().unwrap() + } + + #[track_caller] + fn assert_excluded(patterns: impl IntoIterator, path: &str) { + let filter = filter_with_defaults(patterns); + assert!( + filter.match_directory(SystemPath::new(path), GlobFilterCheckMode::TopDown), + "`{path}` should be excluded" + ); + } + + #[track_caller] + fn assert_not_excluded(patterns: impl IntoIterator, path: &str) { + let filter = filter_with_defaults(patterns); + assert!( + !filter.match_directory(SystemPath::new(path), GlobFilterCheckMode::TopDown), + "`{path}` should not be excluded" + ); + } + + /// A negation that names the directory itself re-includes it, whichever way it's spelled. + #[test] + fn negation_re_includes_a_default_excluded_directory() { + assert_excluded([], "/project/dist"); + + assert_not_excluded(["!dist"], "/project/dist"); + assert_not_excluded(["!dist/"], "/project/dist"); + assert_not_excluded(["!**/dist/"], "/project/dist"); + assert_not_excluded(["!./dist"], "/project/dist"); + assert_not_excluded(["!/project/dist"], "/project/dist"); + } + + /// Patterns are anchored at the project root, so an unqualified negation doesn't reach a + /// nested directory of the same name. `**/` does. + #[test] + fn negation_of_a_nested_directory_needs_a_wildcard() { + assert_excluded(["!dist"], "/project/pkg/dist"); + assert_not_excluded(["!**/dist/"], "/project/pkg/dist"); + } + + /// `dist/**` names the contents of `dist`, not `dist` itself, so negating it leaves the + /// directory excluded and the contents out of reach. + #[test] + fn negating_the_contents_leaves_the_directory_excluded() { + assert_excluded(["!dist/**"], "/project/dist"); + } +} diff --git a/crates/ty_project/src/glob/portable.rs b/crates/ty_project/src/glob/portable.rs index 3e2cd568cb..7e249411b3 100644 --- a/crates/ty_project/src/glob/portable.rs +++ b/crates/ty_project/src/glob/portable.rs @@ -147,8 +147,15 @@ impl<'a> PortableGlobPattern<'a> { } if pattern.starts_with('/') { + // An already-rooted pattern needs no anchoring, but it does need its `!` back: + // without it `!/root/src` reads as an exclude of `/root/src`, the opposite of + // what was written. return AbsolutePortableGlobPattern { - absolute: pattern.to_string(), + absolute: if negated { + format!("!{pattern}") + } else { + pattern.to_string() + }, relative: self.pattern.to_string(), }; } @@ -231,6 +238,64 @@ impl AbsolutePortableGlobPattern { pub(crate) fn relative(&self) -> &str { &self.relative } + + /// Returns `true` if this is a negated (`!`-prefixed) exclude pattern. + pub(crate) fn is_negated(&self) -> bool { + self.absolute.starts_with('!') + } + + /// The deepest directory that has to be walked for this pattern to match anything. + /// + /// A glob can only match paths that start with the components before its first wildcard, + /// so those components bound how deep the pattern can reach. `/root/dist/**` can only ever + /// match something inside `/root/dist`, and `/root/dist/generated.py` can only match a file + /// in that same directory, so both need `/root/dist` to be walked. A pattern that names a + /// path outright needs only that path's parent, which is why `/root/dist` itself returns + /// `/root`: re-including `dist` doesn't require walking into it. + /// + /// Returns `None` for a pattern that isn't anchored to any directory at all, such as + /// `**/dist/` before it's made absolute. + pub(crate) fn required_directory(&self) -> Option<&SystemPath> { + let pattern = self.absolute.strip_prefix('!').unwrap_or(&self.absolute); + + let wildcard_start = pattern + .split('/') + .scan(0usize, |offset, component| { + let start = *offset; + *offset += component.len() + 1; + Some((start, component)) + }) + .find(|(_, component)| is_wildcard_component(component)) + .map(|(start, _)| start); + + match wildcard_start { + // Everything before the first wildcard is literal, so that prefix is the deepest + // directory the pattern can be anchored to. The trailing `/` is dropped with it. + Some(start) => { + let prefix = &pattern[..start.saturating_sub(1)]; + (!prefix.is_empty()).then(|| SystemPath::new(prefix)) + } + // No wildcard at all: the pattern names one path, so only its parent is walked. + None => SystemPath::new(pattern).parent(), + } + } +} + +/// Returns `true` if `component` contains an unescaped glob metacharacter. +fn is_wildcard_component(component: &str) -> bool { + let mut chars = component.chars(); + + while let Some(c) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '*' | '?' | '[' => return true, + _ => {} + } + } + + false } #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -393,6 +458,45 @@ mod tests { assert_absolute_path("./src", "/root", "/root/src"); } + /// A pattern that's already rooted skips anchoring, but it still has to keep its `!`: + /// dropping it turns a re-include into an exclude of the same path. + #[test] + fn negated_absolute_pattern_keeps_its_negation() { + assert_absolute_path("!/src", "/root", "!/src"); + assert_absolute_path("!./src", "/root", "!/root/src"); + } + + #[track_caller] + fn assert_required_directory( + pattern: &str, + relative_to: impl AsRef, + expected: &str, + ) { + let pattern = PortableGlobPattern::parse(pattern, PortableGlobKind::Exclude) + .unwrap() + .into_absolute(relative_to); + assert_eq!( + pattern.required_directory().map(SystemPath::as_str), + (!expected.is_empty()).then_some(expected) + ); + } + + #[test] + fn required_directory() { + // A pattern that names one path only needs that path's parent walked. + assert_required_directory("!dist", "/root", "/root"); + assert_required_directory("!dist/", "/root", "/root"); + assert_required_directory("!dist/generated.py", "/root", "/root/dist"); + // Everything before the first wildcard bounds how deep the pattern can reach. + assert_required_directory("!dist/**", "/root", "/root/dist"); + assert_required_directory("!dist/*.py", "/root", "/root/dist"); + assert_required_directory("!**/dist/", "/root", "/root"); + // An escaped `*` is a literal, not a wildcard. + assert_required_directory(r"!dist/\*/x.py", "/root", r"/root/dist/\*"); + // Nothing to anchor to. + assert_required_directory("**/dist/", "", ""); + } + #[test] #[cfg(windows)] fn absolute_pattern_windows() { diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index 15aca23cd7..39c2d5cf60 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -1,5 +1,9 @@ use crate::Db; -use crate::glob::{ExcludeFilter, IncludeExcludeFilter, IncludeFilter, PortableGlobKind}; +use crate::GlobFilterCheckMode; +use crate::glob::{ + AbsolutePortableGlobPattern, ExcludeFilter, IncludeExcludeFilter, IncludeFilter, + PortableGlobKind, +}; use crate::metadata::python_version::SupportedPythonVersion; use crate::metadata::settings::{OverrideSettings, SrcSettings}; @@ -1015,7 +1019,13 @@ pub struct SrcOptions { /// - `**/venv/` /// /// You can override any default exclude by using a negated pattern. For example, - /// to re-include `dist` use `exclude = ["!dist"]` + /// to re-include `dist` use `exclude = ["!dist"]`, or `exclude = ["!**/dist/"]` to + /// re-include every `dist` directory rather than only the one at the project root. + /// + /// A negated pattern can only re-include something that is still walked, so it cannot + /// reach into a directory that is itself excluded. `exclude = ["!dist/generated.py"]` + /// re-includes nothing, because the walk stops at `dist`. Re-include the directory + /// first: `exclude = ["!**/dist/", "**/dist/**", "!**/dist/generated.py"]` #[option( default = r#"null"#, value_type = r#"list[str]"#, @@ -1052,6 +1062,7 @@ impl SrcOptions { self.exclude.as_ref(), DEFAULT_SRC_EXCLUDES, GlobFilterContext::SrcRoot, + diagnostics, )?; let files = IncludeExcludeFilter::new(include, exclude); @@ -1276,6 +1287,7 @@ fn build_exclude_filter( exclude_patterns: Option<&RangedValue>>, default_patterns: &[&str], context: GlobFilterContext, + diagnostics: &mut Vec, ) -> Result> { use crate::glob::{ExcludeFilterBuilder, PortableGlobPattern}; @@ -1290,12 +1302,19 @@ fn build_exclude_filter( }); } + // Held on to so that, once the filter is built, every negation can be checked against the + // whole pattern set — including the negations that come after it. + let mut negations = Vec::new(); + // Add user-specified excludes if let Some(exclude_patterns) = exclude_patterns { for exclude in exclude_patterns { - exclude + let pattern = exclude .absolute(project_root, system, PortableGlobKind::Exclude) - .and_then(|pattern| Ok(excludes.add(&pattern)?)) + .and_then(|pattern| { + excludes.add(&pattern)?; + Ok(pattern) + }) .map_err(|err| { let diagnostic = OptionDiagnostic::new( DiagnosticId::InvalidGlob, @@ -1311,10 +1330,14 @@ fn build_exclude_filter( err, ) })?; + + if pattern.is_negated() { + negations.push((exclude, pattern)); + } } } - excludes.build().map_err(|_| { + let filter = excludes.build().map_err(|_| { let diagnostic = OptionDiagnostic::new( DiagnosticId::InvalidGlob, format!( @@ -1328,7 +1351,63 @@ fn build_exclude_filter( "Please open an issue on the ty repository \ and share the patterns that caused the error.", ))) - }) + })?; + + for (exclude, pattern) in negations { + let Some(blocked_by) = unreachable_negation_cause(&filter, &pattern) else { + continue; + }; + + // Paths in a configuration file read better relative to the project they configure. + let blocked_by_display = blocked_by.strip_prefix(project_root).unwrap_or(blocked_by); + + let mut diagnostic = OptionDiagnostic::new( + DiagnosticId::UnreachableExcludeNegation, + format!("Negated pattern `{exclude}` has no effect"), + Severity::Warning, + ) + .sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format!("`{blocked_by_display}` is excluded, so nothing inside it is ever reached"), + )); + + if let Some(name) = blocked_by.file_name() { + diagnostic = diagnostic.sub(SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format!("Re-include the directory first by adding `!**/{name}/`"), + )); + } + + if let Some(source_file) = exclude.value().source().file() + && let Ok(file) = system_path_to_file(db, source_file) + { + diagnostic = diagnostic.with_annotation(Some( + Annotation::primary(Span::from(file).with_optional_range(exclude.value().range())) + .message("This pattern can never match"), + )); + } + + diagnostics.push(diagnostic); + } + + Ok(filter) +} + +/// Returns the excluded directory that stops `negation` from ever re-including anything. +/// +/// A negation only takes effect if the walk actually reaches the paths it matches, and the walk +/// stops at the first excluded directory. So the pattern is dead if any directory it has to be +/// reached through is excluded — the shallowest such directory is the one reported, because that +/// is where the walk really stops. +fn unreachable_negation_cause<'a>( + filter: &ExcludeFilter, + negation: &'a AbsolutePortableGlobPattern, +) -> Option<&'a SystemPath> { + negation + .required_directory()? + .ancestors() + .filter(|directory| filter.match_directory(directory, GlobFilterCheckMode::TopDown)) + .last() } /// Context for filter operations, used in error messages @@ -2600,6 +2679,7 @@ impl ToOverride for RangedValue { self.exclude.as_ref(), &[], GlobFilterContext::Overrides, + diagnostics, )?; let files = IncludeExcludeFilter::new(include, exclude); diff --git a/ty.schema.json b/ty.schema.json index 6e281b01aa..878b371c38 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -2454,7 +2454,7 @@ "type": "object", "properties": { "exclude": { - "description": "A list of file and directory patterns to exclude from type checking.\n\nPatterns follow a syntax similar to `.gitignore`:\n\n- `./src/` matches only a directory\n- `./src` matches both files and directories\n- `src` matches files or directories named `src`\n- `*` matches any (possibly empty) sequence of characters (except `/`).\n- `**` matches zero or more path components.\n This sequence **must** form a single path component, so both `**a` and `b**` are invalid and will result in an error.\n A sequence of more than two consecutive `*` characters is also invalid.\n- `?` matches any single character except `/`\n- `[abc]` matches any character inside the brackets. Character sequences can also specify ranges of characters, as ordered by Unicode,\n so e.g. `[0-9]` specifies any character between `0` and `9` inclusive. An unclosed bracket is invalid.\n- `!pattern` negates a pattern (undoes the exclusion of files that would otherwise be excluded)\n\nAll paths are anchored relative to the project root (`src` only\nmatches `/src` and not `/test/src`).\nTo exclude any directory or file named `src`, use `**/src` instead.\n\nBy default, ty excludes commonly ignored directories:\n\n- `**/.bzr/`\n- `**/.direnv/`\n- `**/.eggs/`\n- `**/.git/`\n- `**/.git-rewrite/`\n- `**/.hg/`\n- `**/.mypy_cache/`\n- `**/.nox/`\n- `**/.pants.d/`\n- `**/.pytype/`\n- `**/.ruff_cache/`\n- `**/.svn/`\n- `**/.tox/`\n- `**/.venv/`\n- `**/__pypackages__/`\n- `**/_build/`\n- `**/buck-out/`\n- `**/dist/`\n- `**/node_modules/`\n- `**/venv/`\n\nYou can override any default exclude by using a negated pattern. For example,\nto re-include `dist` use `exclude = [\"!dist\"]`", + "description": "A list of file and directory patterns to exclude from type checking.\n\nPatterns follow a syntax similar to `.gitignore`:\n\n- `./src/` matches only a directory\n- `./src` matches both files and directories\n- `src` matches files or directories named `src`\n- `*` matches any (possibly empty) sequence of characters (except `/`).\n- `**` matches zero or more path components.\n This sequence **must** form a single path component, so both `**a` and `b**` are invalid and will result in an error.\n A sequence of more than two consecutive `*` characters is also invalid.\n- `?` matches any single character except `/`\n- `[abc]` matches any character inside the brackets. Character sequences can also specify ranges of characters, as ordered by Unicode,\n so e.g. `[0-9]` specifies any character between `0` and `9` inclusive. An unclosed bracket is invalid.\n- `!pattern` negates a pattern (undoes the exclusion of files that would otherwise be excluded)\n\nAll paths are anchored relative to the project root (`src` only\nmatches `/src` and not `/test/src`).\nTo exclude any directory or file named `src`, use `**/src` instead.\n\nBy default, ty excludes commonly ignored directories:\n\n- `**/.bzr/`\n- `**/.direnv/`\n- `**/.eggs/`\n- `**/.git/`\n- `**/.git-rewrite/`\n- `**/.hg/`\n- `**/.mypy_cache/`\n- `**/.nox/`\n- `**/.pants.d/`\n- `**/.pytype/`\n- `**/.ruff_cache/`\n- `**/.svn/`\n- `**/.tox/`\n- `**/.venv/`\n- `**/__pypackages__/`\n- `**/_build/`\n- `**/buck-out/`\n- `**/dist/`\n- `**/node_modules/`\n- `**/venv/`\n\nYou can override any default exclude by using a negated pattern. For example,\nto re-include `dist` use `exclude = [\"!dist\"]`, or `exclude = [\"!**/dist/\"]` to\nre-include every `dist` directory rather than only the one at the project root.\n\nA negated pattern can only re-include something that is still walked, so it cannot\nreach into a directory that is itself excluded. `exclude = [\"!dist/generated.py\"]`\nre-includes nothing, because the walk stops at `dist`. Re-include the directory\nfirst: `exclude = [\"!**/dist/\", \"**/dist/**\", \"!**/dist/generated.py\"]`", "anyOf": [ { "$ref": "#/definitions/Array_of_string" From 9865af06c575c3e6022fcd84a145a0b00a1044b8 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:35:25 +1000 Subject: [PATCH 3/7] an emitted type answers as the interpreted definition it stands for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a class inside a package reported the bare last component as its `__module__`, because the module's name and its C symbol prefix had been conflated into one string. they are two names: cpython reads a class's module off the front of its `tp_name`, while the extension's file name and `PyInit_` symbol take the last component only. `dataclasses` looks that name up in `sys.modules`, so two modules failed to import at all. the artefact is now written at its module's place in the output tree rather than flat — where two members sharing a last component silently overwrote each other, and no package member's artefact was importable under the name it was compiled as. the file name and the init symbol follow different rules, and only the symbol is the last component: `a/b/__init__.py` needs the file `__init__.so` and the symbol `PyInit_b`. a class-level constant was copied off the twin *after* the twin's own decorators had run over it, so `@dataclass` deleting a `field(init=False)` left nothing to copy, and setting a bare default lost `repr=False` — a silent wrong answer no sweep can see. each class body is now captured before its decorators run, by giving the module body its own `__builtins__` carrying a wrapped `__build_class__`. per-namespace rather than process-wide, because this backend builds for free-threaded interpreters where another thread could be importing at the same time. a decorator ran twice — once in the twin's source, once at init — so any decorator with an effect performed it twice. module-level functions and classes now have theirs taken out of the twin's source, with the definitions the module body reads during import declining instead, since in that window the name would hold an undecorated definition. a *method's* still runs twice: blanking cannot work there, because the class construction consumes the decorator's effect — `ABCMeta.__new__` computes `__abstractmethods__` from the namespace the body left, so a blanked `@abstractmethod` changes the class the twin builds. also: a decorator written as a chain of attributes is lowered rather than declined; a class whose type slots publish more than its body wrote keeps its decorator's decline, turning two silent miscompiles into honest refusals; two definitions of one name in a scope decline rather than emitting one C symbol twice and failing the build; and `by run --compiled` passes the tree root, where it had been silently running every package `__init__` interpreted. --- crates/basedpython/Cargo.lock | 1 + crates/by_build/src/annotate.rs | 2 +- crates/by_build/src/lib.rs | 78 +- crates/by_build/src/toolchain.rs | 19 +- crates/by_build/tests/differential.rs | 9957 ++++++++++++++++--------- crates/by_build/tests/end_to_end.rs | 108 +- crates/by_codegen_c/src/lib.rs | 982 ++- crates/by_ir/src/builder.rs | 7 +- crates/by_ir/src/function.rs | 374 +- crates/by_ir/src/lib.rs | 4 +- crates/by_ir/src/ops.rs | 25 + crates/by_ir/src/print.rs | 14 +- crates/by_ir/src/verify.rs | 17 +- crates/by_irbuild/Cargo.toml | 1 + crates/by_irbuild/src/closures.rs | 34 +- crates/by_irbuild/src/lib.rs | 1448 +++- crates/by_irbuild/src/single_file.rs | 2 +- crates/by_irbuild/src/tests.rs | 1360 +++- crates/by_opt/src/coalesce.rs | 2 +- crates/by_opt/src/copy_propagation.rs | 4 +- crates/by_opt/src/dead_registers.rs | 9 +- crates/by_opt/src/fold.rs | 2 +- crates/by_opt/src/infallible.rs | 4 +- crates/by_opt/src/lib.rs | 4 +- crates/by_opt/src/refcount.rs | 2 +- crates/by_opt/src/str_append.rs | 2 +- crates/by_opt/src/str_item_compare.rs | 2 +- crates/by_opt/src/unswitch.rs | 2 +- crates/by_rt/include/by.h | 856 ++- crates/by_rt/src/lib.rs | 31 + 30 files changed, 11329 insertions(+), 4024 deletions(-) diff --git a/crates/basedpython/Cargo.lock b/crates/basedpython/Cargo.lock index 8ff93b7e8a..44bbd09001 100644 --- a/crates/basedpython/Cargo.lock +++ b/crates/basedpython/Cargo.lock @@ -251,6 +251,7 @@ dependencies = [ "by_ir", "ruff_db", "ruff_python_ast", + "ruff_python_parser", "ruff_python_stdlib", "ruff_text_size", "thin-vec", diff --git a/crates/by_build/src/annotate.rs b/crates/by_build/src/annotate.rs index de691cd079..306c4f3bf5 100644 --- a/crates/by_build/src/annotate.rs +++ b/crates/by_build/src/annotate.rs @@ -15,7 +15,7 @@ use by_ir::print::print_function; /// render the report for a lowered module pub fn report(module: &ModuleIr) -> String { - let mut out = format!("# {}\n", module.name); + let mut out = format!("# {}\n", module.name.dotted()); let native: Vec<&Function> = module.all_functions().collect(); let _ = writeln!( diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index ea00a8d7bf..3e47e542b7 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -34,7 +34,7 @@ pub struct Artifact { /// C, and invoke the platform compiler pub fn build_source( source: &str, - module_name: &str, + module_name: impl Into, toolchain: &Toolchain, out_dir: &Path, options: &Options, @@ -87,7 +87,7 @@ pub fn emit_lowered( /// be written, so `--emit-c-only` and a real build report the same layout pub fn emit_source( source: &str, - module_name: &str, + module_name: impl Into, out_dir: &Path, options: &Options, ) -> Result { @@ -109,21 +109,32 @@ fn emit_verified(module: &ModuleIr, out_dir: &Path, options: &Options) -> Result .with_context(|| format!("could not create {}", out_dir.display()))?; fs::write(out_dir.join(by_rt::BY_H_NAME), by_rt::BY_H)?; - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let source_path = out_dir.join(format!("{last}.c")); + let source_path = out_dir.join(module.name.relative_path(".c")); + create_parent(&source_path)?; fs::write(&source_path, by_codegen_c::emit_module(module)) .with_context(|| format!("could not write {}", source_path.display()))?; Ok(Built { artifact: Artifact { source: source_path, - extension: out_dir.join(format!("{last}.so")), + extension: out_dir.join(module.name.relative_path(".so")), annotation: write_annotation(module, out_dir, options)?, }, declined: module.declined.clone(), }) } +/// make the directory `path` is to be written into +/// +/// an artefact sits at its module's own place in the output tree, so a package +/// member's directory may not exist yet +fn create_parent(path: &Path) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + fs::create_dir_all(parent).with_context(|| format!("could not create {}", parent.display())) +} + /// what a build is allowed to leave interpreted #[derive(Debug, Clone, Default)] pub struct Options { @@ -173,8 +184,8 @@ fn write_annotation( if !options.annotate { return Ok(None); } - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let path = out_dir.join(format!("{last}.annotated")); + let path = out_dir.join(module.name.relative_path(".annotated")); + create_parent(&path)?; fs::write(&path, annotate::report(module)) .with_context(|| format!("could not write {}", path.display()))?; Ok(Some(path)) @@ -191,7 +202,7 @@ fn render_declines<'a>(declines: impl Iterator, options: &Options, version: Option<(u8, u8)>, ) -> Result { @@ -217,9 +228,11 @@ fn finish( // or a debugger lands on source somebody wrote. a caller that knows the real // path sets this itself — the bare module name is the fallback if module.lines.is_none() { - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let path = module + .name + .relative_path(&format!(".{}", options.language.extension())); module.lines = Some(by_ir::function::LineTable::new( - format!("{last}.{}", options.language.extension()), + path.display().to_string(), source, )); } @@ -260,21 +273,25 @@ fn finish( // the version has to be the *interpreter's*, because this python runs inside // the extension at import time — emitting syntax the interpreter cannot parse // makes the whole module fail to load, taking every function with it - if options.language == by_irbuild::Language::Python { - // a `.py` source is already what runs: it is its own fallback - module.fallback_source = Some(source.to_string()); - return Ok(module); - } - let mut config = options.fallback.clone().unwrap_or_default(); - if let Some((major, minor)) = version - && let Ok(parsed) = format!("{major}.{minor}").parse() - { - config.min_version = parsed; - } - let transpiled = by_transforms::transpile(source, &config).map_err(|error| { - anyhow::anyhow!("could not transpile for the interpreted fallback: {error}") - })?; - module.fallback_source = Some(transpiled); + // a `.py` source is already what runs: it is its own fallback + let twin = if options.language == by_irbuild::Language::Python { + source.to_string() + } else { + let mut config = options.fallback.clone().unwrap_or_default(); + if let Some((major, minor)) = version + && let Ok(parsed) = format!("{major}.{minor}").parse() + { + config.min_version = parsed; + } + by_transforms::transpile(source, &config).map_err(|error| { + anyhow::anyhow!("could not transpile for the interpreted fallback: {error}") + })? + }; + // a decorator module init applies to the native definition would otherwise run here + // too, over the twin's — once for each definition rather than once for the name + let twin = by_irbuild::without_init_decorators(&twin, &module) + .map_err(|error| anyhow::anyhow!("could not prepare the interpreted fallback: {error}"))?; + module.fallback_source = Some(twin); Ok(module) } @@ -304,14 +321,17 @@ pub fn build_module(module: &ModuleIr, toolchain: &Toolchain, out_dir: &Path) -> fs::create_dir_all(out_dir) .with_context(|| format!("could not create {}", out_dir.display()))?; + // the header stays at the root of the output tree, and the root is what is put + // on the include path — so one copy serves every artefact however deep its + // package goes let header = out_dir.join(by_rt::BY_H_NAME); let header_changed = write_if_changed(&header, by_rt::BY_H.as_bytes())?; - let last = module.name.rsplit('.').next().unwrap_or(&module.name); - let source = out_dir.join(format!("{last}.c")); + let source = out_dir.join(module.name.relative_path(".c")); + create_parent(&source)?; let source_changed = write_if_changed(&source, by_codegen_c::emit_module(module).as_bytes())?; - let extension = out_dir.join(toolchain.extension_file_name(&module.name)); + let extension = out_dir.join(toolchain.extension_path(&module.name)); // the C compiler is by far the slowest step, and the emitted C is a faithful // function of the optimized BIR — so identical C means an identical compile. // keying on the C rather than on the `.by` is what makes a comment-only edit, @@ -470,7 +490,7 @@ mod tests { // returning a float from an int function builder.terminate(Terminator::Return(Value::Float(1.0))); let module = ModuleIr { - name: "bad".to_string(), + name: by_ir::ModuleName::new("bad"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_build/src/toolchain.rs b/crates/by_build/src/toolchain.rs index d5a171c60d..ea51242ed4 100644 --- a/crates/by_build/src/toolchain.rs +++ b/crates/by_build/src/toolchain.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::process::Command; use anyhow::{Context, Result, bail}; +use by_ir::ModuleName; use serde::Deserialize; /// everything needed to compile and link an extension for one interpreter @@ -175,10 +176,10 @@ impl Toolchain { }) } - /// the file name an extension for `module` must have to be importable - pub fn extension_file_name(&self, module: &str) -> String { - let last = module.rsplit('.').next().unwrap_or(module); - format!("{last}{}", self.ext_suffix) + /// where an extension for `module` must sit, relative to the root of an output + /// tree, to be importable under that name + pub fn extension_path(&self, module: &ModuleName) -> PathBuf { + module.relative_path(&self.ext_suffix) } } @@ -234,11 +235,15 @@ mod tests { } #[test] - fn the_extension_name_uses_only_the_last_module_component() { + fn the_extension_path_mirrors_the_module_tree() { let toolchain = Toolchain::from_probe("python3", SAMPLE).unwrap(); assert_eq!( - toolchain.extension_file_name("pkg.app"), - "app.cpython-313-darwin.so" + toolchain.extension_path(&ModuleName::new("pkg.app")), + PathBuf::from("pkg/app.cpython-313-darwin.so") + ); + assert_eq!( + toolchain.extension_path(&ModuleName::package("pkg")), + PathBuf::from("pkg/__init__.cpython-313-darwin.so") ); } diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 4f18de5268..a045a8e33a 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -560,7 +560,7 @@ fn agree_in( language, ..Options::default() }; - let built = match build_source(source, &module, &toolchain, &compiled_dir, &options) { + let built = match build_source(source, module.as_str(), &toolchain, &compiled_dir, &options) { Ok(built) => built, Err(error) => { // only an *absent* toolchain is a skip. anything else is the compiler @@ -2429,743 +2429,1447 @@ def stacked(n: int) -> int: ); } +/// a decorator is evaluated once, and what it did on the way happened once +/// +/// python evaluates it where the `def` stands. the interpreted twin is what stands there +/// and module init evaluates it again over the compiled definition, so `mark` appended +/// twice while the name it left behind was right either way — the whole of the defect was +/// the second append. `mark` returns what it was handed, so nothing but `marked` can +/// show it #[test] -fn string_literals_do_not_leak() { - // `gc.get_objects()` cannot see this: `str` is not GC-tracked, so a leaked - // literal is invisible to an object-count check. the refcount of the returned - // literal is the measurement that works +fn a_decorator_runs_once() { + agree( + "decoratoronce", + "\ +marked: list[int] = [] + + +def mark(f: object) -> object: + marked.append(1) + return f + + +@mark +def counted() -> int: + return 1 + + +@mark +def counted_twice() -> int: + return 2 +", + &["m.counted()", "m.counted_twice()", "m.marked"], + ); +} + +/// a definition the module *reads* keeps its decorator, and declines +/// +/// taking the decorator out of the twin's source leaves the name holding an undecorated +/// definition from the twin's `def` until module init reaches it — a window nothing can +/// see unless the module's own body looks. `AT_IMPORT` looks directly and `alias` keeps +/// what it found, and both of them would otherwise hold what `double` never wrapped +#[test] +fn a_decorated_definition_the_module_reads_declines() { + agree_with_declines( + "decoratorread", + "\ +def double(f) -> object: + def wrapper() -> int: + return f() * 2 + return wrapper + + +@double +def one() -> int: + return 1 + + +AT_IMPORT = one() +alias = one +", + &["m.one()", "m.AT_IMPORT", "m.alias()"], + ); +} + +/// the source both class-decorator tests below compile +/// +/// `mark` hands back what it was given and records only the name, so the binding is right +/// however many times it ran and `seen` is the one thing that can show a second run +const MARKED_CLASSES: &str = "\ +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Marked: + def g(self) -> int: + return 1 + + +@mark +class Second: + def h(self) -> int: + return 2 +"; + +/// a class's decorator is evaluated once, and what it did on the way happened once +/// +/// python runs it where the `class` statement stands. the interpreted twin is what stands +/// there and module init ran it again over the namespace entry the compiled type had +/// taken, so `seen` read `['Marked', 'Second', 'Marked', 'Second']` where python reads +/// `['Marked', 'Second']` — and the class each name ended up bound to was right either +/// way, which is what made it silent +#[test] +fn a_class_decorator_runs_once() { + agree_python( + "classdecoratoronce", + MARKED_CLASSES, + &["m.Marked().g()", "m.Second().h()", "m.seen"], + ); +} + +#[test] +fn a_decorated_class_is_the_compiled_type() { + // the counts above are answered identically by a class that fell back to its + // interpreted definition, so they cannot say which build answered. + // `method_descriptor` can: a compiled type holds one where the interpreted class + // holds a plain function let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_strleak"); + let dir = std::env::temp_dir().join("by_diff_classdecoratoronce_t"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def classify(n: int) -> str: - scratch = \"x\" + \"y\" - if n < 0: - return \"neg\" - return scratch -"; - if build_source( - source, - "by_diff_strleak", + let built = match build_source( + MARKED_CLASSES, + "by_diff_classdecoratoronce_t", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import sys, by_diff_strleak as m\n\ - before = sys.getrefcount(m.classify(-1))\n\ - for _ in range(20000):\n m.classify(-1)\n m.classify(1)\n\ - after = sys.getrefcount(m.classify(-1))\n\ - print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", + "import by_diff_classdecoratoronce_t as m\n\ + print(type(m.Marked.__dict__['g']).__name__,\n\ + \x20 type(m.Second.__dict__['h']).__name__)\n\ + print(m.seen)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n['Marked', 'Second']" ); - assert_eq!(out, "stable"); } +/// a decorated class the module *reads* keeps its decorator, and declines +/// +/// taking the decorator out of the twin's source leaves the interpreted definition +/// standing undecorated from its `class` statement until module init reaches it. `TABLE` +/// looks in that window and keeps what it found, so the list would hold a class the +/// module's own name no longer means — `TABLE[0] is Held` would answer `False` #[test] -fn a_concatenated_operand_keeps_the_callers_reference() { - // an operand handed over to a growing concatenation is a reference the frame - // did not own — the caller's count goes down and the object is freed under it. - // the two builds return the same string either way, so only the count says so +fn a_decorated_class_the_module_reads_declines() { + agree_python_with_declines( + "classdecoratorread", + "\ +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Held: + def value(self) -> int: + return 1 + + +TABLE = [Held] +", + &["m.Held().value()", "m.seen", "m.TABLE[0] is m.Held"], + ); +} + +/// a class named in an *unevaluated* annotation is not read, and keeps compiling +/// +/// `from __future__ import annotations` makes `Held` in that signature a string nothing +/// evaluates, so the module never holds the undecorated definition and the decorator can +/// still move to init. without the future import this is `TABLE = [Held]` again +#[test] +fn a_decorated_class_named_in_a_deferred_annotation_still_compiles() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_strhold"); + let dir = std::env::temp_dir().join("by_diff_classdecoratorannotation"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -def join(a: str, b: str) -> str: - return a + b +from __future__ import annotations -def grow(seed: str, n: int) -> str: - out = seed - i = 0 - while i < n: - out = out + \"x\" - i = i + 1 - return out +seen = [] + + +def mark(o): + seen.append(o.__name__) + return o + + +@mark +class Held: + def value(self) -> int: + return 1 + + +def through(h: Held) -> int: + return h.value() "; - if build_source( + let built = match build_source( source, - "by_diff_strhold", + "by_diff_classdecoratorannotation", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import sys, by_diff_strhold as m\n\ - held = 'a' * 40\n\ - before = sys.getrefcount(held)\n\ - for _ in range(20000):\n m.join(held, 'y')\n m.grow(held, 3)\n\ - after = sys.getrefcount(held)\n\ - print('stable' if after == before else f'moved {before}->{after}')\n", + "import by_diff_classdecoratorannotation as m\n\ + print(type(m.Held.__dict__['value']).__name__, m.through(m.Held()))\n\ + print(m.seen)\n", ); - assert_eq!(out, "stable"); + assert_eq!(out, "method_descriptor 1\n['Held']"); } +/// a decorator written as a chain of attributes, which is what the ir grew an +/// expression to hold +/// +/// the ir carried a single `String` and codegen emitted one interned lookup of it, so +/// `functools.cache` was a name to find whole in the module dict — it was declined +/// rather than compiled, which is why it never raised. every step of a path is a *read*, +/// which is what makes evaluating it at module init mean what it meant where the `def` +/// stood #[test] -fn native_classes_agree() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree( - "classes", +fn a_decorator_written_as_a_path_agrees() { + agree_python( + "pathdeco", "\ -data class Point: - x: int - y: int +import abc +import functools - def total(self) -> int: - return self.x + self.y +CALLS = [] - def scaled(self, k: int) -> int: - return self.total() * k -data class Named: - label: str - count: int +class Wrappers: + @staticmethod + def tag(cls: type) -> type: + cls.tag = 'seen' + return cls - def shout(self) -> str: - return self.label + \"!\" -def make(a: int, b: int) -> object: - return Point(a, b) +@functools.cache +def cached(n: int) -> int: + CALLS.append(n) + return n * 2 + + +class Marks: + @abc.abstractmethod + def area(self) -> int: + return 3 + + +@Wrappers.tag +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +def probe() -> int: + return cached(4) + cached(4) ", &[ - "m.Point(3, 4).total()", - "m.Point(3, 4).scaled(10)", - "[m.Point(a, b).total() for a in (0, -1, 10 ** 20) for b in (0, 5)]", - "m.make(2, 3).total()", - "(m.Point(1, 2).x, m.Point(1, 2).y)", - "m.Named('hi', 1).shout()", - "m.Named('hi', 1).label", - // the argument *count* is checked in both builds - "[(type(e).__name__) for e in [_capture(m.Point, 1)]]", + // the wrapper the decorator returned is what the name holds, and it is + // reached from inside the module as well as outside + "m.probe()", + "m.cached(9)", + // and it ran once, so the second call was a cache hit + "[m.probe(), m.CALLS]", + "type(m.cached).__name__", + // a path off something that is not a module resolves the same way + "m.Held(2).read()", + "m.Held.tag", + // a method's decorator is resolved out of the module namespace too + "m.Marks.area.__isabstractmethod__", + "m.Marks().area()", ], ); } +/// the compiled leg answers the path-decorated definitions +/// +/// the differential legs agree whichever one answered, so a decorator test passes with +/// the codegen path switched off. this is where it is pinned #[test] -fn a_native_constructor_checks_its_argument_types() { - // a documented delta, not an equality: a compiled field is an unboxed - // `ByTagged`, so a `str` cannot be stored there and the check is mandatory. - // `@dataclass` does not enforce annotations at runtime, so the interpreted - // twin accepts it — the same difference `--soundness all` would close +fn a_path_decorated_definition_is_the_compiled_one() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_ctorcheck"); + let dir = std::env::temp_dir().join("by_diff_pathdecolive"); let _ = std::fs::remove_dir_all(&dir); - let source = "data class Point:\n x: int\n y: int\n"; - if build_source( + let source = "\ +import abc +import functools + + +class Wrappers: + @staticmethod + def tag(cls: type) -> type: + cls.tag = 'seen' + return cls + + +@functools.cache +def cached(n: int) -> int: + return n * 2 + + +class Marks: + @abc.abstractmethod + def area(self) -> int: + return 3 + + +@Wrappers.tag +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n +"; + let built = match build_source( source, - "by_diff_ctorcheck", + "by_diff_pathdecolive", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - let out = run( + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( &python, &dir, - "import by_diff_ctorcheck as m\n\ - try:\n m.Point('a', 1)\n\ - except TypeError as e:\n print('TypeError:', e)\n\ - else:\n print('accepted a str')\n", + "import by_diff_pathdecolive as m\n\ + print(type(m.cached).__name__, type(m.cached.__wrapped__).__name__)\n\ + print(type(m.Marks.area).__name__, type(m.Held.read).__name__)\n\ + print(m.cached(4), m.Marks.area.__isabstractmethod__, m.Held.tag)\n", + ); + assert_eq!( + out, + "_lru_cache_wrapper builtin_function_or_method\n\ + method method_descriptor\n\ + 8 True seen" ); - assert_eq!(out, "TypeError: expected int, got str"); } +/// a decorator that is a call keeps its decline +/// +/// python calls `mark('x')` where the `def` stands. module-level code is not compiled, +/// so the only moment init has is the end of the module — by which time the interpreted +/// twin has already made that call. making it again would be a second one, in the wrong +/// place, with whatever it did on the way happening twice #[test] -fn a_native_class_has_a_fixed_layout() { +fn a_decorator_that_is_a_call_declines() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_layout"); + let dir = std::env::temp_dir().join("by_diff_calldeco"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int - y: int +MADE = [] + + +def mark(label: str): + MADE.append(label) + + def apply(fn): + fn.label = label + return fn + + return apply + + +@mark('x') +def f(n: int) -> int: + return n + 1 "; - if build_source( + let built = match build_source( source, - "by_diff_layout", + "by_diff_calldeco", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // no `__dict__`, and no attribute outside the declared set + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("run it a second time")), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_layout as m\n\ - p = m.Point(1, 2)\n\ - print(hasattr(p, '__dict__'))\n\ - try:\n p.extra = 1\n\ - except AttributeError:\n print('AttributeError')\n\ - else:\n print('accepted an undeclared attribute')\n", + "import by_diff_calldeco as m\n\ + print(m.f(1), m.f.label, m.MADE, type(m.f).__name__)\n", ); - assert_eq!(out, "False\nAttributeError"); + // the factory ran once, where it was written, and the interpreted definition is + // what the name holds + assert_eq!(out, "2 x ['x'] function"); } +/// a decorator rooted at a name the class body bound keeps its decline +/// +/// `@total.setter` reads the property the body bound above it. a decorator is resolved +/// out of the *module* namespace at init, where there is no such name — the lookup would +/// raise `NameError` and take the whole extension's import with it. that shape also +/// writes two `def`s of one name, which is its own decline and answers first, so +/// `Rooted` stands beside it with the same root and distinct names #[test] -fn a_native_class_instance_does_not_leak() { +fn a_decorator_rooted_in_the_class_body_declines() { let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_classleak"); + let dir = std::env::temp_dir().join("by_diff_classrooteddeco"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str - n: int -"; - if build_source( - source, - "by_diff_classleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // `tp_dealloc` has to release each refcounted field - let out = run( - &python, - &dir, - "import sys, by_diff_classleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n m.Holder(label, 1)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", - ); - assert_eq!(out, "stable"); -} +class Box: + def __init__(self, n: int) -> None: + self._n = n -#[test] -fn a_declined_function_still_exists_and_behaves_the_same() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 11))) { - return; - } - // the interpreted fallback is what makes coverage total: a construct with no - // native lowering costs speed in that one place and nothing anywhere else - agree_with_declines( - "fallback", - "\ -def fast(a: int) -> int: - return a * 2 + @property + def total(self) -> int: + return self._n -def slow(n: int) -> int: - out = n - try: - pass - except* ValueError: - out = 0 - return out -", - &[ - "m.fast(21)", - "m.slow(21)", - // the declined one is a plain python function in both builds - "type(m.slow).__name__", - ], - ); -} + @total.setter + def total(self, value: int) -> None: + self._n = value -#[test] -fn a_float_module_imports_without_any_extra_runtime_module() { - // `float` transpiles through `JustFloat`, which the lazy-import pass binds - // locally. transpiling with `lazy_imports` off instead emits - // `from ty_extensions import JustFloat`, and the extension then fails to - // import at all — the embedded fallback runs at module init - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_floatimport"); - let _ = std::fs::remove_dir_all(&dir); - let source = "def area(r: float) -> float:\n return 3.0 * r * r\n"; - if build_source( + +class Rooted: + def wrap(fn): + return fn + + @wrap + def value(self) -> int: + return 3 +"; + let built = match build_source( source, - "by_diff_floatimport", + "by_diff_classrooteddeco", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // nothing but the extension itself is on the path + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let reasons = |needle: &str| { + built + .declined + .iter() + .any(|declined| declined.reason.contains(needle)) + }; + assert!( + reasons("`wrap` is bound by the class body"), + "declined: {:?}", + built.declined + ); + assert!( + reasons("`total` is defined more than once"), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_floatimport as m\nprint(m.area(2.0))\n", + "import by_diff_classrooteddeco as m\n\ + b = m.Box(1)\n\ + b.total = 7\n\ + print(b.total, type(m.Box.total).__name__, m.Rooted().value())\n", ); - assert_eq!(out, "12.0"); + assert_eq!(out, "7 property 3"); } -#[test] -fn a_compiled_function_is_a_c_function_object() { - // one of the few places the two builds are *meant* to differ: a natively - // compiled function is a builtin, not a python function. recorded in - // plan.md#semantic-deltas rather than asserted as equal +/// a class whose type slots publish more than its body wrote keeps its decorator's +/// decline +/// +/// python reaches `<=` through `tp_richcompare`, one slot behind all six comparisons — +/// so an emitted type that writes `__lt__` publishes `__le__` as well, answering +/// `NotImplemented`. `functools.total_ordering` reads exactly that: it saw `__le__` +/// already there, filled in nothing, and `a <= b` raised where the interpreted class +/// answered `True`. that was a live wrong answer for the plain-name spelling before the +/// path spelling could reach it at all +#[test] +fn a_class_decorator_over_a_partly_filled_slot_declines() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_cfunc"); + let dir = std::env::temp_dir().join("by_diff_partialslot"); let _ = std::fs::remove_dir_all(&dir); - let source = "def f(a: int) -> int:\n return a\n"; - if build_source( + let source = "\ +import functools + + +@functools.total_ordering +class Ranked: + def __init__(self, n: int) -> None: + self.n = n + + def __eq__(self, other: object) -> bool: + return isinstance(other, Ranked) and self.n == other.n + + def __lt__(self, other: object) -> bool: + return self.n < other.n +"; + let built = match build_source( source, - "by_diff_cfunc", + "by_diff_partialslot", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("publishes `__le__`")), + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_cfunc as m\nprint(type(m.f).__name__)\n", + "import by_diff_partialslot as m\n\ + print(m.Ranked(1) <= m.Ranked(2), m.Ranked(3) > m.Ranked(2))\n\ + print(type(m.Ranked.__init__).__name__)\n", ); - assert_eq!(out, "builtin_function_or_method"); + assert_eq!(out, "True True\nfunction"); } #[test] -fn module_level_code_runs_at_import() { - agree( - "modulelevel", +fn a_call_to_a_decorated_function_from_the_same_module_agrees() { + // the module namespace holds what the decorator returned; the native entry holds + // what it was handed. a call written inside the module used to reach the entry, so + // `caller(1)` answered 2 compiled and 4 interpreted with nothing said about it + agree_python( + "decoratedcallee", "\ -LIMIT = 7 +def double(fn): + def inner(x: int) -> int: + return fn(x) * 2 + return inner -def under(n: int) -> int: - return n -def limit() -> int: - return 7 +@double +def f(x: int) -> int: + return x + 1 + + +def caller(x: int) -> int: + return f(x) + + +def plain(x: int) -> int: + return x + 1 + + +def other(x: int) -> int: + return plain(x) ", - &["m.LIMIT", "m.limit()", "m.under(3)"], + &[ + "m.caller(1)", + "m.f(1)", + "m.other(1)", + // the decorator ran exactly once on whatever the name holds, rather than + // once per call site + "[m.caller(1), m.caller(1), m.f(1)]", + ], ); } #[test] -fn no_any_turns_a_gradual_decline_into_an_error() { - let Some((_, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_noany"); - let _ = std::fs::remove_dir_all(&dir); +fn a_construction_of_a_decorated_class_from_the_same_module_agrees() { + // a construction is written against the *name*, and a class decorator is what the + // name then holds. allocating the emitted layout instead skipped the decorator: + // `probe()` answered 3 compiled and 103 interpreted + agree_python( + "decoratedclassctor", + "\ +class Other: + def __init__(self, x: int) -> None: + self.x = x + 100 - let source = "\ -def precise(a: int) -> int: - return a + 1 -def loose(a) -> None: - pass -"; - // by default a gradual parameter compiles — it lands on `object` - let built = build_source( - source, - "by_diff_noany", - &toolchain, - &dir, - &Options::default(), +def swap(cls): + return Other + + +@swap +class C: + def __init__(self, x: int) -> None: + self.x = x + + +def probe(x: int) -> int: + return C(x).x +", + &[ + "m.probe(3)", + "m.C(3).x", + "m.Other(1).x", + // the name is what a construction resolves, from either side of the module + "m.C is m.Other", + ], ); - match built { - Ok(built) => assert!(built.declined.is_empty(), "{:?}", built.declined), - Err(error) => { - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - } +} - // `--no-any` refuses instead, and names the function - let error = build_source( - source, - "by_diff_noany", - &toolchain, - &dir, - &Options { - no_any: true, - ..Options::default() - }, - ) - .unwrap_err() - .to_string(); - assert!(error.contains("no-any"), "{error}"); - assert!(error.contains("loose"), "{error}"); - assert!(error.contains("`a` is gradual"), "{error}"); - // a fully typed function is not blamed - assert!(!error.contains("precise"), "{error}"); +#[test] +fn a_method_modifier_is_not_looked_up_as_a_name() { + // a modifier reaches the ast as a decorator with no `@`. it was emitted as a name + // to look up in the module namespace at init, and there is no such name — so the + // extension built cleanly and then failed to import outright with `NameError: name + // 'override' is not defined`, taking every function in the module with it. + // + // `static` is the sharper case and is not here: it declines, because a method's + // slot zero is forced to the receiver and `staticmethod` says it is not one + agree( + "methodmodifier", + "\ +class Box: + abstract def area(self) -> int: + return 7 + +def probe() -> int: + return Box().area() +", + &[ + "m.probe()", + "m.Box().area()", + // the modifier became `abstractmethod`, so the method carries the same + // marker the interpreted twin carries — not a name nobody bound + "getattr(m.Box.area, '__isabstractmethod__', None)", + ], + ); } #[test] -fn require_native_rejects_any_decline_at_all() { - // a different question from `--no-any`: `list[int]` is not gradual, it is a - // type the compiler does not represent *yet*, so only this flag catches it - let Some((_, toolchain)) = environment() else { +fn a_static_or_class_method_answers_the_same_through_the_class_and_through_an_instance() { + // slot zero used to be forced to the receiver for every method, and these two say + // it is not one — so the compiled `Box.make(3)` bound `3` to a `Box` and raised + // `unbound method Box.make() needs an argument` at its first call. + // + // each is reached four ways: through the class, through an instance, from a method + // of the same class, and from a module-level function. all four go through the + // *descriptor* the type publishes, so a wrong convention shows up in every one + agree_python( + "staticmethod", + "\ +class Box: + @staticmethod + def make(x: int) -> int: + return x + 7 + + @classmethod + def named(cls, y: int) -> str: + return cls.__name__ + str(y) + + def n(self) -> int: + return 1 + + def inside(self) -> int: + return Box.make(1) + len(Box.named(2)) + + +class Alt: + def __init__(self, v: int) -> None: + self.v = v + + @classmethod + def of(cls, v: int) -> \"Alt\": + return cls(v) + + +def probe() -> int: + return Box.make(3) +", + &[ + "m.probe()", + "m.Box.make(3)", + "m.Box().make(3)", + "m.Box.named(2)", + "m.Box().named(2)", + "m.Box().inside()", + "m.Box.make(x=3)", + // a class method is what an alternative constructor is written as, and + // `cls(v)` has to reach the class it was called on + "m.Alt.of(4).v", + "type(m.Alt.of(4)).__name__", + // `__self__` is the class for a class method and nothing at all for a + // static one, on either build + "m.Box.named.__self__ is m.Box", + "getattr(m.Box.make, '__self__', None)", + ], + ); +} + +#[test] +fn a_static_or_class_method_is_the_compiled_one() { + // neither `agree` can say which build answered — a class that declines answers + // identically out of its interpreted definition. `type(C.__dict__['m'])` cannot + // say either: it is `staticmethod` on both legs. + // + // the *descriptor* is what differs. a compiled static or class method is reached + // through a `PyCFunction`, so `type(C.m)` is `builtin_function_or_method` where the + // interpreted leg has a plain `function` or a bound `method` — and the class + // method's dict entry is a `classmethod_descriptor` rather than a `classmethod` + let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_reqnative"); + let dir = std::env::temp_dir().join("by_diff_staticmethod_which"); let _ = std::fs::remove_dir_all(&dir); - // precisely typed and still declines: `except*` has no lowering let source = "\ -def precise(a: int) -> int: - return a + 1 +class Box: + @staticmethod + def make(x: int) -> int: + return x + 7 -def grouped(a: int) -> int: - out = a - try: - pass - except* ValueError: - out = 0 - return out + @classmethod + def named(cls, y: int) -> str: + return cls.__name__ + str(y) "; - // `--no-any` is satisfied: nothing here is gradual - match build_source( + let built = match build_source( source, - "by_diff_reqnative", + "by_diff_staticmethod_which", &toolchain, &dir, &Options { - no_any: true, + language: by_irbuild::Language::Python, ..Options::default() }, ) { - Ok(built) => assert_eq!(built.declined.len(), 1, "{:?}", built.declined), + Ok(built) => built, Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); eprintln!("skipping: no working C toolchain ({error})"); return; } - } - - // `--require-native` is not - let error = build_source( - source, - "by_diff_reqnative", - &toolchain, + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, &dir, - &Options { - require_native: true, - ..Options::default() - }, - ) - .unwrap_err() - .to_string(); - assert!(error.contains("require-native"), "{error}"); - assert!(error.contains("`except*`"), "{error}"); + "import by_diff_staticmethod_which as m\n\ + print(type(m.Box.make).__name__, type(m.Box.__dict__['make']).__name__)\n\ + print(type(m.Box.named).__name__, type(m.Box.__dict__['named']).__name__)\n\ + print(m.Box.make(3), m.Box.named(2))\n", + ); + assert_eq!( + out, + "builtin_function_or_method staticmethod\n\ + builtin_function_or_method classmethod_descriptor\n\ + 10 Box2" + ); } #[test] -fn no_any_accepts_a_fully_typed_module() { - let Some((_, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_noany_ok"); - let _ = std::fs::remove_dir_all(&dir); - let source = "def f(a: int, b: int) -> int:\n return a * b\n"; - match build_source( - source, - "by_diff_noany_ok", - &toolchain, - &dir, - &Options { - no_any: true, - ..Options::default() - }, - ) { - Ok(built) => assert!(built.declined.is_empty()), - Err(error) => eprintln!("skipping: no working C toolchain ({error})"), - } +fn a_static_or_class_method_on_a_class_built_through_its_metaclass_answers() { + // a class on a base out of this module may be built by *calling* its metaclass, + // and that construction puts the method table into a namespace rather than onto a + // type — so the descriptor is one the runtime builds. building the plain kind for + // either of these would hand the function the wrong receiver + agree_python( + "staticmethodmeta", + "\ +import collections.abc + + +class Sized(collections.abc.Sized): + def __len__(self) -> int: + return 3 + + @staticmethod + def tag() -> str: + return \"sized\" + + @classmethod + def kind(cls) -> str: + return cls.__name__ +", + &[ + "m.Sized.tag()", + "m.Sized().tag()", + "m.Sized.kind()", + "m.Sized().kind()", + "len(m.Sized())", + "type(m.Sized).__name__", + ], + ); } #[test] -fn optimized_output_still_agrees_with_the_interpreter() { - // copy propagation and infallibility both rewrite the IR — the differential - // legs are what say the rewrites preserved the program - agree( - "optimized", +fn a_static_method_the_boundary_hands_over_reaches_the_plain_function() { + // a boundary that cannot establish a parameter hands the whole call to the + // interpreted twin, and for a *method* that twin is taken off the class with the + // receiver put back in front of it. a static method has no receiver — `self` holds + // nothing at all — and the twin taken off the class is already the plain function + // the `staticmethod` wraps, so prepending anything would have handed it NULL. + // + // both reasons to hand over are here: `float` admits an `int`, and a default that + // is not an immediate is one object every call has to share + agree_python( + "staticmethoddefer", "\ -def area(r: float) -> float: - scaled = r * r - return 3.5 * scaled +DEFAULT = [1, 2] -def compose(a: float, b: float) -> float: - x = area(a) - y = area(b) - return x + y + +class Box: + @staticmethod + def half(x: float) -> float: + return x / 2 + + @staticmethod + def counted(xs=DEFAULT) -> int: + return len(xs) + + @staticmethod + def nests(n: int) -> int: + def inner(k: int) -> int: + return k + n + return inner(1) + + +def nests(n: int) -> int: + def inner(k: int) -> int: + return k + n + 100 + return inner(1) ", &[ - "m.area(2.0)", - "m.compose(1.0, 2.0)", - "[m.compose(a, a) for a in (0.0, -1.5, 3.25)]", + "m.Box.half(5.0)", + // the int arrives where a `double` was compiled, so this is the call that + // goes back to the interpreted definition + "m.Box.half(5)", + "m.Box().half(5)", + "m.Box.counted()", + "m.Box.counted([1])", + "m.Box.counted() and m.Box.counted() is not None", + // the object the default holds is the module's, shared by every call + "m.Box.counted.__self__ if hasattr(m.Box.counted, '__self__') else None", + // a nested function lives on a generated class named after the frame that + // makes it, and these two frames are both called `nests` + "m.Box.nests(3)", + "m.nests(3)", ], ); } #[test] -fn a_declined_function_is_reported_with_a_reason() { - let Some((_, toolchain)) = environment() else { +fn a_class_method_the_boundary_would_hand_over_declines() { + // the twin a method's boundary hands over to is taken off the interpreted class, + // and for a class method python has already *bound* it — to that class, not to the + // one in slot zero. handing it the class as well would give the body two of them + let Some((python, toolchain)) = environment() else { return; }; - let dir: PathBuf = std::env::temp_dir().join("by_diff_declined"); + let dir = std::env::temp_dir().join("by_diff_classmethod_defer"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def fast(a: int) -> int: - return a + 1 +DEFAULT = [1, 2] -def slow(a: int) -> None: - try: - pass - except* ValueError: - pass + +class Box: + @classmethod + def counted(cls, xs=DEFAULT) -> int: + return len(xs) "; - let Ok(built) = build_source( + let built = match build_source( source, - "by_diff_declined", + "by_diff_classmethod_defer", &toolchain, &dir, - &Options::default(), - ) else { - eprintln!("skipping: no working C toolchain"); - return; + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } }; - assert_eq!(built.declined.len(), 1); - assert_eq!(built.declined[0].name, "slow"); assert!( - built.declined[0].reason.contains("`except*`"), - "{:?}", - built.declined[0] + built.declined.iter().any(|declined| declined + .reason + .contains("already bound to the interpreted class")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_classmethod_defer as m\n\ + print(m.Box.counted(), m.Box.counted([1]))\n", ); + assert_eq!(out, "2 1"); } #[test] -fn field_access_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree( - "fields", +fn a_global_a_frame_assigns_reaches_the_module_namespace() { + // the write half of a `global` declaration. there was no op for it, so the + // assignment bound a local and the module's name kept its old value — a wrong + // answer rather than a missing one, and the reason this shape was declined. + // + // `agree_python` asserts nothing declined, and + // `a_compiled_frame_is_what_reaches_the_module_namespace` below is what says the + // native function is the one python calls + agree_python( + "globalwrite", "\ -data class Point: - x: int - y: int +inited = False +counter = 0 - def total(self) -> int: - return self.x + self.y - def shift(self, d: int) -> int: - self.x = self.x + d - return self.x +def init() -> None: + global inited, counter + inited = True + counter += 1 -data class Line: - a: Point - b: Point - def span(self) -> int: - return self.b.x - self.a.x +def bump(n: int) -> int: + global counter + counter = counter + n + return counter +", + &[ + // read from *outside* after a write from a compiled frame: a register + // write is invisible here, which is the whole of the bug + "(m.inited, m.counter)", + "(m.init(), m.inited, m.counter)", + "(m.init(), m.counter)", + "(m.bump(5), m.counter)", + // and the interpreted world's own write is what the compiled frame's + // next read has to see, since both are the one dict + "(setattr(m, 'counter', 100), m.bump(1), m.counter)", + ], + ); +} - def retarget(self, p: Point) -> int: - self.b = p - return self.span() +#[test] +fn a_global_a_frame_assigns_is_read_back_in_that_same_frame() { + // the other half, and the one a write alone can get wrong in the opposite + // direction: if the write reaches the namespace while a later read in the same + // frame still resolves a register, the two halves stop agreeing with each other. + // + // this is `mimetypes` in miniature — a constructor that initializes the module the + // first time it runs, and an `init` whose flag never landed, so the second + // construction called it again and the compiled leg recursed until the stack ran out + agree_python( + "globalselfread", + "\ +inited = False +log: list[str] = [] -def sum_of(p: Point) -> int: - return p.x + p.y -def bump_twice(p: Point) -> int: - p.shift(1) - p.shift(2) - return p.x +class C: + def __init__(self) -> None: + if not inited: + init() + self.x = 1 + + +def init() -> None: + global inited + log.append('init') + inited = True + # the read the write has to be visible to, in this frame and through `C` + if inited: + C() + + +def flip() -> str: + global inited + inited = not inited + # written, read, written again: three answers out of one place + first = inited + inited = not inited + return f'{first} {inited}' ", &[ - "m.Point(3, 4).total()", - "m.sum_of(m.Point(3, 4))", - "m.Point(3, 4).shift(10)", - "m.bump_twice(m.Point(0, 0))", - "m.Line(m.Point(1, 2), m.Point(10, 20)).span()", - "m.Line(m.Point(1, 2), m.Point(10, 20)).a.y", - "m.Line(m.Point(1, 2), m.Point(10, 20)).retarget(m.Point(100, 0))", - // a write through the python-visible setter, then a native read - "[(p := m.Point(1, 2), setattr(p, 'x', 9), p.total())[-1]]", - "[m.sum_of(m.Point(a, b)) for a in (0, -1, 10 ** 20) for b in (0, 5)]", + "(m.C().x, m.inited, m.log)", + "m.flip()", + "(m.flip(), m.inited)", ], ); } #[test] -fn a_field_setter_checks_its_value() { - // the same documented delta as the constructor: an unboxed field cannot hold - // the wrong representation, and `@dataclass` does not check assignments +fn a_global_a_frame_deletes_leaves_the_name_unbound() { + // `del x` under a `global x` unbinds the module's name, and reading it afterwards + // is a `NameError` — which is not the `KeyError` deleting from a dict raises, nor + // what a register could ever report + agree_python( + "globaldelete", + "\ +value = 1 + + +def drop() -> str: + global value + del value + try: + return repr(value) + except NameError as error: + return f'NameError: {error}' + + +def again() -> str: + global value + try: + del value + except NameError as error: + return f'NameError: {error}' + return 'deleted' + + +def restore(n: int) -> int: + global value + value = n + return value +", + &[ + "m.drop()", + "m.again()", + "(m.restore(7), m.value)", + "(m.drop(), m.again())", + ], + ); +} + +#[test] +fn a_global_a_nested_frame_declares_is_not_the_enclosing_local_of_that_name() { + // the enclosing frame binds a local `seen` and the nested one declares `seen` + // global, so they are two different places. deciding captures without consulting + // the declaration makes the closure read and write the enclosing local instead, + // and both the module's name and the local then answer wrongly + agree_python( + "globalnested", + "\ +seen = 0 +tally = 0 + + +def outer(n: int) -> str: + seen = n + + def inner() -> int: + global seen + seen = 5 + return seen + + return f'{inner()} {seen}' + + +def only_reads(n: int) -> str: + # the nested frame declares the name and never writes it, so nothing about *it* + # says the enclosing local is the wrong place — only the declaration does + seen = n + + def peek() -> int: + global seen + return seen + + return f'{peek()} {seen}' + + +def declared_out_here(n: int) -> str: + # and the mirror: the enclosing frame declares it and writes it, so it has no + # register for the nested frame to capture even though the name looks local + global tally + tally = n + + def peek() -> int: + return tally + + return f'{peek()} {tally}' + + +def shadow(n: int) -> int: + # no declaration: an ordinary local that shadows the module's name + seen = n + return seen +", + &[ + "(m.outer(3), m.seen)", + "(m.only_reads(9), m.seen)", + "(m.declared_out_here(4), m.tally)", + "(m.shadow(9), m.seen)", + ], + ); +} + +#[test] +fn a_global_a_generator_assigns_is_not_one_of_its_state_fields() { + // a generator's locals become fields of the state object, because the frame has to + // survive a suspension. a declared `global` is not one of them — the module + // namespace already outlives every suspension — so it must be kept out of that + // layout as much as out of a register, and each resumption has to write through + agree_python( + "globalgen", + "\ +total = 0 +steps: list[int] = [] + + +def counting(n: int): + global total + for i in range(n): + total = total + i + # the write has to be visible across the suspension, from outside and back + yield total + + +def resets(): + global total + total = 0 + yield total + total = 100 + yield total +", + &[ + "(list(m.counting(4)), m.total)", + "(list(m.counting(3)), m.total)", + // stepped by hand, reading the module's name between resumptions + "[(next(g), m.total) for g in [m.counting(5)] for _ in range(3)]", + "(list(m.resets()), m.total)", + ], + ); +} + +#[test] +fn a_module_level_name_a_global_rebinds_is_found_through_the_namespace() { + // taking the write opened this: a frame can now rebind a name the module *defined*, + // and a call written against that name was reaching the definition directly. so the + // rebound function went on answering with the old body — the same thing a decorator + // does to a name, and it goes in the same set + agree_python( + "globalrebind", + "\ +def base() -> int: + return 1 + + +def other() -> int: + return 99 + + +def calls_base() -> int: + return base() + + +def rebind() -> str: + global base + base = other + return f'{base()} {calls_base()}' + + +def replaces_itself() -> int: + # `pydoc.pager` is this exactly: decide once what to be, rebind the name, then + # call *through the name*. reaching the native entry for that last call re-enters + # the body that just rebound it, and the stack runs out + global replaces_itself + replaces_itself = other + return replaces_itself() +", + &[ + "(m.calls_base(), m.base())", + "m.rebind()", + "(m.calls_base(), m.base())", + "m.replaces_itself()", + "(m.replaces_itself(), m.replaces_itself())", + ], + ); +} + +#[test] +fn a_compiled_frame_is_what_reaches_the_module_namespace() { + // the differential tests above compare two legs, and a leg that fell back to its + // interpreted definition answers exactly as the interpreted leg does — so they + // cannot say *which* build wrote the global. this one can: a module-level function + // python calls through `PyModule_AddFunctions` is a `builtin_function_or_method`, + // and one that fell back is a `function` let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_setcheck"); + let dir = std::env::temp_dir().join("by_diff_globalidentity"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int - y: int +inited = False -frozen data class Fixed: - n: int + +class C: + def __init__(self) -> None: + if not inited: + init() + self.x = 1 + + +def init() -> None: + global inited + inited = True + C() "; - if build_source( + let built = match build_source( source, - "by_diff_setcheck", + "by_diff_globalidentity", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); let out = run( &python, &dir, - "import by_diff_setcheck as m\n\ - p = m.Point(1, 2)\n\ - p.x = 5\n\ - print(p.x)\n\ - for attempt in (lambda: setattr(p, 'x', 'a'), lambda: delattr(p, 'x'), lambda: setattr(m.Fixed(1), 'n', 2)):\n\ - \x20 try:\n attempt()\n\ - \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ - \x20 else:\n print('accepted')\n", + "import by_diff_globalidentity as m\n\ + print(type(m.init).__name__, m.C().x, m.inited)\n", ); - assert_eq!(out, "5\nTypeError\nAttributeError\nAttributeError"); + // and `m.inited` read from out here is the module's own binding, which a register + // write never touched. before there was an op for it, `C()` inside `init` saw the + // old `False` and called `init` again until the stack ran out + assert_eq!(out, "builtin_function_or_method 1 True"); } #[test] -fn a_class_typed_argument_is_checked_at_the_boundary() { - // a compiled parameter typed as a native class is a pointer to its struct, - // so python handing over anything else has to be caught here — the - // alternative is a wild pointer dereference +fn a_declined_function_reads_the_global_a_compiled_one_wrote() { + // the asymmetry the whole thing turns on. a compiled frame and the interpreted + // twin of a *declined* one are the same module, and the twin's `__globals__` is + // the dict the compiled frame binds into — so a write is visible to it at once. + // a register write is visible to nobody, which is why this had to decline let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_argcheck"); + let dir = std::env::temp_dir().join("by_diff_globaltwin"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Point: - x: int +flag = 0 -def read(p: Point) -> int: - return p.x + +def writes(n: int) -> int: + global flag + flag = n + return flag + + +def declines_and_reads() -> str: + # `del` on a plain local has no lowering, so this whole function stays interpreted + tmp = 1 + del tmp + return f'{flag}' "; - if build_source( + let built = match build_source( source, - "by_diff_argcheck", + "by_diff_globaltwin", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // the two legs of the same module: one compiled, one not. if `del` on a local ever + // gains a lowering this stops being true, and the assertion says so rather than + // quietly testing two compiled functions + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + vec!["declines_and_reads"], + "declined: {:?}", + built.declined + ); let out = run( &python, &dir, - "import by_diff_argcheck as m\n\ - print(m.read(m.Point(7)))\n\ - for bad in ('a', 1, None):\n\ - \x20 try:\n m.read(bad)\n\ - \x20 except TypeError as e:\n print(e)\n\ - \x20 else:\n print('accepted')\n", - ); - assert_eq!( - out, - "7\nexpected by_diff_argcheck.Point, got str\n\ - expected by_diff_argcheck.Point, got int\n\ - expected by_diff_argcheck.Point, got NoneType" + "import by_diff_globaltwin as m\n\ + print(type(m.writes).__name__, type(m.declines_and_reads).__name__,\n\ + \x20 m.declines_and_reads(), m.writes(42), m.declines_and_reads(), m.flag)\n", ); + assert_eq!(out, "builtin_function_or_method function 0 42 42 42"); } #[test] -fn a_field_read_does_not_leak() { +fn a_second_decorator_over_a_static_method_declines() { + // the runtime folds the rest of a method's decorators onto the attribute it reads + // back off the finished type — and reading a static method back hands over the + // plain function it wraps, which would be written back as an ordinary method. so + // the pair keeps the decline, and the interpreted definition is what answers let Some((python, toolchain)) = environment() else { return; }; - if !supports(&toolchain, (3, 10)) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - let dir = std::env::temp_dir().join("by_diff_fieldleak"); + let dir = std::env::temp_dir().join("by_diff_staticmethod_stacked"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str +def mark(fn): + fn.marked = True + return fn -def label_of(h: Holder) -> str: - return h.label -data class Nest: - inner: Holder +class Stacked: + @mark + @staticmethod + def both() -> int: + return 1 +"; + let built = match build_source( + source, + "by_diff_staticmethod_stacked", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built + .declined + .iter() + .any(|declined| declined.reason.contains("a second decorator over")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_staticmethod_stacked as m\n\ + print(m.Stacked.both(), type(m.Stacked.__dict__['both']).__name__)\n", + ); + assert_eq!(out, "1 staticmethod"); +} -def inner_label(n: Nest) -> str: - return n.inner.label +#[test] +fn string_literals_do_not_leak() { + // `gc.get_objects()` cannot see this: `str` is not GC-tracked, so a leaked + // literal is invisible to an object-count check. the refcount of the returned + // literal is the measurement that works + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_strleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def classify(n: int) -> str: + scratch = \"x\" + \"y\" + if n < 0: + return \"neg\" + return scratch "; if build_source( source, - "by_diff_fieldleak", + "by_diff_strleak", &toolchain, &dir, &Options::default(), @@ -3175,117 +3879,73 @@ def inner_label(n: Nest) -> str: eprintln!("skipping: no working C toolchain"); return; } - // a field read hands back a *borrowed* reference, so the register that takes - // it must retain — and release. `str` is not gc-tracked, so this measures the - // refcount rather than the object count let out = run( &python, &dir, - "import sys, by_diff_fieldleak as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - n = m.Nest(h)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.label_of(h)\n\ - \x20 m.inner_label(n)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n", + "import sys, by_diff_strleak as m\n\ + before = sys.getrefcount(m.classify(-1))\n\ + for _ in range(20000):\n m.classify(-1)\n m.classify(1)\n\ + after = sys.getrefcount(m.classify(-1))\n\ + print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", ); assert_eq!(out, "stable"); } #[test] -fn a_constructor_result_is_used_natively() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); +fn a_concatenated_operand_keeps_the_callers_reference() { + // an operand handed over to a growing concatenation is a reference the frame + // did not own — the caller's count goes down and the object is freed under it. + // the two builds return the same string either way, so only the count says so + let Some((python, toolchain)) = environment() else { return; - } - agree( - "ctornative", - "\ -data class Point: - x: int - y: int - - def total(self) -> int: - return self.x + self.y - -data class Line: - a: Point - b: Point - -def diag(a: int) -> int: - return Point(a, a).x - -def build(a: int, b: int) -> int: - return Line(Point(a, b), Point(b, a)).b.x - -def widened(a: int) -> object: - return Point(a, a) - -def chained(a: int) -> int: - p = Point(a, a + 1) - q = Point(p.y, p.x) - return q.total() + p.total() -", - &[ - "m.diag(7)", - "m.build(3, 9)", - "m.widened(4).total()", - "m.chained(5)", - "[m.chained(a) for a in (0, -3, 10 ** 20)]", - ], - ); -} + }; + let dir = std::env::temp_dir().join("by_diff_strhold"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def join(a: str, b: str) -> str: + return a + b -#[test] -fn a_declined_callee_does_not_break_the_build() { - // the whole module used to fail to compile: the emitted call named a symbol - // that was never defined - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); +def grow(seed: str, n: int) -> str: + out = seed + i = 0 + while i < n: + out = out + \"x\" + i = i + 1 + return out +"; + if build_source( + source, + "by_diff_strhold", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); return; } - agree_with_declines( - "declinechain", - "\ -def helper[T](a: T) -> T: - return a - -def caller(a: int) -> int: - return helper(a) + a - -data class Point: - x: int - - def bad[T](self, a: T) -> int: - return self.x - -def read(p: Point) -> int: - return p.x - -def alone(a: int) -> int: - return a + 1 -", - &[ - "m.helper(1)", - "m.caller(2)", - "m.read(m.Point(5))", - "m.Point(5).bad(1)", - "m.alone(3)", - ], + let out = run( + &python, + &dir, + "import sys, by_diff_strhold as m\n\ + held = 'a' * 40\n\ + before = sys.getrefcount(held)\n\ + for _ in range(20000):\n m.join(held, 'y')\n m.grow(held, 3)\n\ + after = sys.getrefcount(held)\n\ + print('stable' if after == before else f'moved {before}->{after}')\n", ); + assert_eq!(out, "stable"); } #[test] -fn direct_method_dispatch_agrees() { +fn native_classes_agree() { if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { eprintln!("skipping: `data class` needs python 3.10"); return; } agree( - "direct", + "classes", "\ data class Point: x: int @@ -3297,53 +3957,36 @@ data class Point: def scaled(self, k: int) -> int: return self.total() * k - def shifted(self, d: int) -> int: - self.x = self.x + d - return self.total() - - def label(self) -> str: - return \"p\" - -data class Line: - a: Point - b: Point - - def span(self) -> int: - return self.b.total() - self.a.total() - - def widest(self, other: Line) -> int: - mine = self.span() - theirs = other.span() - if mine > theirs: - return mine - return theirs +data class Named: + label: str + count: int -def drive(a: int, b: int) -> int: - p = Point(a, b) - return p.scaled(2) + p.total() + p.shifted(1) + def shout(self) -> str: + return self.label + \"!\" -def countdown(p: Point, n: int) -> int: - total = 0 - for _ in range(n): - total = total + p.total() - return total +def make(a: int, b: int) -> object: + return Point(a, b) ", &[ - "m.Point(3, 4).scaled(5)", - "m.Point(3, 4).label()", - "m.drive(1, 2)", - "m.countdown(m.Point(2, 3), 4)", - "m.Line(m.Point(1, 1), m.Point(10, 10)).span()", - "m.Line(m.Point(1, 1), m.Point(10, 10)).widest(m.Line(m.Point(0, 0), m.Point(2, 2)))", - "[m.drive(a, a) for a in (0, -7, 10 ** 20)]", - // a python caller reaches the same method through the type object - "[getattr(m.Point(3, 4), 'scaled')(2)]", + "m.Point(3, 4).total()", + "m.Point(3, 4).scaled(10)", + "[m.Point(a, b).total() for a in (0, -1, 10 ** 20) for b in (0, 5)]", + "m.make(2, 3).total()", + "(m.Point(1, 2).x, m.Point(1, 2).y)", + "m.Named('hi', 1).shout()", + "m.Named('hi', 1).label", + // the argument *count* is checked in both builds + "[(type(e).__name__) for e in [_capture(m.Point, 1)]]", ], ); } #[test] -fn a_direct_method_call_does_not_leak() { +fn a_native_constructor_checks_its_argument_types() { + // a documented delta, not an equality: a compiled field is an unboxed + // `ByTagged`, so a `str` cannot be stored there and the check is mandatory. + // `@dataclass` does not enforce annotations at runtime, so the interpreted + // twin accepts it — the same difference `--soundness all` would close let Some((python, toolchain)) = environment() else { return; }; @@ -3351,21 +3994,12 @@ fn a_direct_method_call_does_not_leak() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_methleak"); + let dir = std::env::temp_dir().join("by_diff_ctorcheck"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -data class Holder: - label: str - - def get(self) -> str: - return self.label - -def twice(h: Holder) -> str: - return h.get() + h.get() -"; + let source = "data class Point:\n x: int\n y: int\n"; if build_source( source, - "by_diff_methleak", + "by_diff_ctorcheck", &toolchain, &dir, &Options::default(), @@ -3378,25 +4012,16 @@ def twice(h: Holder) -> str: let out = run( &python, &dir, - "import sys, by_diff_methleak as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.twice(h)\n\ - \x20 h.get()\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n", + "import by_diff_ctorcheck as m\n\ + try:\n m.Point('a', 1)\n\ + except TypeError as e:\n print('TypeError:', e)\n\ + else:\n print('accepted a str')\n", ); - assert_eq!(out, "stable"); + assert_eq!(out, "TypeError: expected int, got str"); } #[test] -fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { - // the borrow pass drops the retain/release pair around `n.inner`. getting it - // wrong is either a leak or a use-after-free, so both directions are checked: - // the label's refcount must be *stable*, and the inner holder must survive - // being dropped from python while a compiled read is in flight +fn a_native_class_has_a_fixed_layout() { let Some((python, toolchain)) = environment() else { return; }; @@ -3404,21 +4029,16 @@ fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_borrow"); + let dir = std::env::temp_dir().join("by_diff_layout"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Holder: - label: str - -data class Nest: - inner: Holder - -def inner_label(n: Nest) -> str: - return n.inner.label +data class Point: + x: int + y: int "; if build_source( source, - "by_diff_borrow", + "by_diff_layout", &toolchain, &dir, &Options::default(), @@ -3428,65 +4048,22 @@ def inner_label(n: Nest) -> str: eprintln!("skipping: no working C toolchain"); return; } + // no `__dict__`, and no attribute outside the declared set let out = run( &python, &dir, - "import sys, by_diff_borrow as m\n\ - label = 'x' * 40\n\ - h = m.Holder(label)\n\ - n = m.Nest(h)\n\ - holder_refs = sys.getrefcount(h)\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.inner_label(n)\n\ - print('label', 'stable' if sys.getrefcount(label) == before else 'moved')\n\ - print('holder', 'stable' if sys.getrefcount(h) == holder_refs else 'moved')\n\ - del h\n\ - print(m.inner_label(n)[:4])\n", - ); - assert_eq!(out, "label stable\nholder stable\nxxxx"); -} - -#[test] -fn calling_a_callable_value_agrees() { - agree( - "callvalue", - "\ -def apply(f: object, a: int) -> object: - return f(a) - -def apply2(f: object, a: int, b: int) -> object: - return f(a, b) - -def indirect(a: int) -> object: - fn = abs - return fn(a) - -def shadowed(len: object, s: str) -> object: - return len(s) - -def nothing(f: object) -> object: - return f() -", - &[ - "m.apply(abs, -5)", - "m.apply(str, 5)", - "m.apply2(max, 3, 9)", - "m.indirect(-7)", - // a parameter shadowing a builtin has to win - "m.shadowed(lambda s: 'shadowed', 'abc')", - "m.nothing(dict)", - // and a value that is not callable raises the same way - "[type(e).__name__ for e in [_capture(m.apply, 3, 4)]]", - "[type(e).__name__ for e in [_capture(m.nothing, None)]]", - ], + "import by_diff_layout as m\n\ + p = m.Point(1, 2)\n\ + print(hasattr(p, '__dict__'))\n\ + try:\n p.extra = 1\n\ + except AttributeError:\n print('AttributeError')\n\ + else:\n print('accepted an undeclared attribute')\n", ); + assert_eq!(out, "False\nAttributeError"); } #[test] -fn a_borrow_survives_a_finalizer_that_runs_a_collection() { - // the borrow pass may only skip the retain where nothing can run in between. - // a `__del__` firing during the window would be the way to catch it wrong +fn a_native_class_instance_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; @@ -3494,23 +4071,16 @@ fn a_borrow_survives_a_finalizer_that_runs_a_collection() { eprintln!("skipping: `data class` needs python 3.10"); return; } - let dir = std::env::temp_dir().join("by_diff_finalizer"); + let dir = std::env::temp_dir().join("by_diff_classleak"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -data class Inner: +data class Holder: label: str - -data class Outer: - inner: Inner - -def read_then_call(o: Outer, f: object) -> str: - held = o.inner.label - f() - return held + o.inner.label + n: int "; if build_source( source, - "by_diff_finalizer", + "by_diff_classleak", &toolchain, &dir, &Options::default(), @@ -3520,171 +4090,65 @@ def read_then_call(o: Outer, f: object) -> str: eprintln!("skipping: no working C toolchain"); return; } + // `tp_dealloc` has to release each refcounted field let out = run( &python, &dir, - "import gc, by_diff_finalizer as m\n\ - class Noisy:\n\ - \x20 def __del__(self):\n gc.collect()\n\ - o = m.Outer(m.Inner('alpha'))\n\ - def churn():\n\ - \x20 Noisy()\n\ - \x20 o.inner = m.Inner('bravo')\n\ - for _ in range(5000):\n\ - \x20 m.read_then_call(o, churn)\n\ - print(m.read_then_call(o, lambda: None))\n", - ); - assert_eq!(out, "bravobravo"); -} - -#[test] -fn reading_a_global_as_a_value_agrees() { - agree( - "globalread", - "\ -LIMIT = 10 - -def limit() -> object: - return LIMIT - -def builtin_alias(a: int) -> object: - fn = abs - return fn(a) - -def missing() -> object: - return not_defined_anywhere -", - &[ - "m.limit()", - "m.builtin_alias(-9)", - "[type(e).__name__ for e in [_capture(m.missing)]]", - // a rebound global is observed, because the read is not cached - "[(setattr(m, 'LIMIT', 99), m.limit())[-1]]", - ], + "import sys, by_diff_classleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n m.Holder(label, 1)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after <= before + 2 else f'leaked {before}->{after}')\n", ); + assert_eq!(out, "stable"); } #[test] -fn closures_agree() { +fn a_declined_function_still_exists_and_behaves_the_same() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 11))) { + return; + } + // the interpreted fallback is what makes coverage total: a construct with no + // native lowering costs speed in that one place and nothing anywhere else agree_with_declines( - "closures", + "fallback", "\ -def make_adder(n: int) -> object: - def add(a: int) -> int: - return a + n - return add - -def make_pair(a: int, b: str) -> object: - def describe(times: int) -> str: - return b * times + str(a) - return describe - -def helper(a: int) -> int: - def double(x: int) -> int: - return x * 2 - return double(a) + double(a) - -def compose(f: object, g: object) -> object: - def both(n: int) -> object: - return f(g(n)) - return both - -def counted(n: int) -> int: - def step(a: int) -> int: - return a + n - total = 0 - for i in range(4): - total = step(total) - return total +def fast(a: int) -> int: + return a * 2 -def used_early(a: int) -> int: - if a > 0: - return later(a) - def later(x: int) -> int: - return x - return later(a) +def slow(n: int) -> int: + out = n + try: + pass + except* ValueError: + out = 0 + return out ", &[ - "m.make_adder(5)(3)", - // two closures from one function must have independent environments - "[(m.make_adder(1), m.make_adder(100))[0](0), m.make_adder(100)(0)]", - "m.make_pair(7, 'ab')(2)", - "m.helper(4)", - "m.compose(abs, lambda n: n - 10)(3)", - "m.counted(2)", - "[m.make_adder(a)(a) for a in (0, -3, 10 ** 20)]", - // the closure is a real callable python can inspect and pass around - "callable(m.make_adder(1))", - "sorted([3, 1, 2], key=m.make_adder(0))", - // and a wrong argument count still raises - "[type(e).__name__ for e in [_capture(m.make_adder(1))]]", - // python raises here, and the interpreted fallback is what reports it - "[type(e).__name__ for e in [_capture(m.used_early, 5)]]", + "m.fast(21)", + "m.slow(21)", + // the declined one is a plain python function in both builds + "type(m.slow).__name__", ], ); } #[test] -fn a_closure_does_not_leak_its_environment() { +fn a_float_module_imports_without_any_extra_runtime_module() { + // `float` transpiles through `JustFloat`, which the lazy-import pass binds + // locally. transpiling with `lazy_imports` off instead emits + // `from ty_extensions import JustFloat`, and the extension then fails to + // import at all — the embedded fallback runs at module init let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_closureleak"); + let dir = std::env::temp_dir().join("by_diff_floatimport"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def make(label: str) -> object: - def get(times: int) -> str: - return label * times - return get -"; + let source = "def area(r: float) -> float:\n return 3.0 * r * r\n"; if build_source( source, - "by_diff_closureleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the environment holds the captured `str`, and releasing the closure has to - // release the environment, which releases the field - let out = run( - &python, - &dir, - "import sys, by_diff_closureleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.make(label)(1)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n\ - held = m.make(label)\n\ - print(sys.getrefcount(label) > before)\n\ - del held\n\ - print(sys.getrefcount(label) == before)\n", - ); - assert_eq!(out, "stable\nTrue\nTrue"); -} - -#[test] -fn a_closure_environment_is_not_visible_in_the_module() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_envhidden"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def make(n: int) -> object: - def get() -> int: - return n - return get -"; - if build_source( - source, - "by_diff_envhidden", + "by_diff_floatimport", &toolchain, &dir, &Options::default(), @@ -3694,36 +4158,29 @@ def make(n: int) -> object: eprintln!("skipping: no working C toolchain"); return; } + // nothing but the extension itself is on the path let out = run( &python, &dir, - "import by_diff_envhidden as m\n\ - print([n for n in dir(m) if 'env' in n])\n\ - print(m.make(3)())\n", + "import by_diff_floatimport as m\nprint(m.area(2.0))\n", ); - assert_eq!(out, "[]\n3"); + assert_eq!(out, "12.0"); } #[test] -fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { - // the exception edge is a CFG edge, and the refcount pass used not to follow it - // — so everything the `try` body had written leaked on the exceptional path +fn a_compiled_function_is_a_c_function_object() { + // one of the few places the two builds are *meant* to differ: a natively + // compiled function is a builtin, not a python function. recorded in + // plan.md#semantic-deltas rather than asserted as equal let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_handlerleak"); + let dir = std::env::temp_dir().join("by_diff_cfunc"); let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def guarded(words: list[str], index: int) -> str: - held = \"held\" + words[0] - try: - return held + words[index] - except IndexError: - return held -"; + let source = "def f(a: int) -> int:\n return a\n"; if build_source( source, - "by_diff_handlerleak", + "by_diff_cfunc", &toolchain, &dir, &Options::default(), @@ -3736,259 +4193,298 @@ def guarded(words: list[str], index: int) -> str: let out = run( &python, &dir, - "import sys, by_diff_handlerleak as m\n\ - words = ['x' * 40]\n\ - label = words[0]\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 m.guarded(words, 9)\n\ - after = sys.getrefcount(label)\n\ - print('stable' if after == before else f'leaked {before}->{after}')\n\ - print(m.guarded(words, 0)[:8])\n", + "import by_diff_cfunc as m\nprint(type(m.f).__name__)\n", ); - assert_eq!(out, "stable\nheldxxxx"); + assert_eq!(out, "builtin_function_or_method"); } #[test] -fn a_mutable_capture_agrees() { - // python closes over the *variable*, so all of these depend on both frames seeing - // one cell — a copy at `def` time would give different answers for every one +fn module_level_code_runs_at_import() { agree( - "cells", + "modulelevel", "\ -def counter() -> (() -> int): - n = 0 - def get() -> int: - return n - n = 1 - return get - -def bumper() -> (() -> int): - n = 0 - def bump() -> int: - nonlocal n - n = n + 1 - return n - return bump - -def loop_closures() -> list[object]: - out = [] - i = 0 - while i < 3: - def show() -> int: - return i - out.append(show) - i = i + 1 - return out +LIMIT = 7 -def shared_pair(start: int) -> list[object]: - def read() -> int: - return start - def write(v: int) -> int: - nonlocal start - start = v - return start - return [read, write] +def under(n: int) -> int: + return n -def accumulate(values: list[int]) -> int: - total = 0 - def add(v: int) -> int: - nonlocal total - total = total + v - return total - for v in values: - add(v) - return total +def limit() -> int: + return 7 ", - &[ - "m.counter()()", - "[(b := m.bumper(), b(), b(), b())[1:]]", - "[f() for f in m.loop_closures()]", - // one cell: the write through one closure is visible through the other - "[(p := m.shared_pair(5), p[0](), p[1](9), p[0]())[1:]]", - "m.accumulate([1, 2, 3])", - "[m.accumulate([a, a]) for a in (0, -4, 10 ** 20)]", - ], + &["m.LIMIT", "m.limit()", "m.under(3)"], ); } #[test] -fn reading_a_cell_before_it_is_written_raises_the_way_python_does() { - // a cell starts unset, and NULL has to read back as an error rather than a zero - agree_with_declines( - "cellunset", - "\ -def early() -> object: - def get() -> int: - return n - out = get - n = 1 - return out +fn no_any_turns_a_gradual_decline_into_an_error() { + let Some((_, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_noany"); + let _ = std::fs::remove_dir_all(&dir); -def early_call() -> object: - def get() -> int: - return n - result = _capture_local(get) - n = 1 - return result + let source = "\ +def precise(a: int) -> int: + return a + 1 -def _capture_local(f: object) -> object: - try: - return f() - except NameError as e: - return type(e).__name__ -", - &[ - // reading it after the write is fine - "m.early()()", - // reading it before is `UnboundLocalError`, which is a `NameError` - "m.early_call()", - ], +def loose(a) -> None: + pass +"; + // by default a gradual parameter compiles — it lands on `object` + let built = build_source( + source, + "by_diff_noany", + &toolchain, + &dir, + &Options::default(), ); + match built { + Ok(built) => assert!(built.declined.is_empty(), "{:?}", built.declined), + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + } + + // `--no-any` refuses instead, and names the function + let error = build_source( + source, + "by_diff_noany", + &toolchain, + &dir, + &Options { + no_any: true, + ..Options::default() + }, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("no-any"), "{error}"); + assert!(error.contains("loose"), "{error}"); + assert!(error.contains("`a` is gradual"), "{error}"); + // a fully typed function is not blamed + assert!(!error.contains("precise"), "{error}"); } #[test] -fn a_shared_cell_does_not_leak() { - let Some((python, toolchain)) = environment() else { +fn require_native_rejects_any_decline_at_all() { + // a different question from `--no-any`: `list[int]` is not gradual, it is a + // type the compiler does not represent *yet*, so only this flag catches it + let Some((_, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_cellleak"); + let dir = std::env::temp_dir().join("by_diff_reqnative"); let _ = std::fs::remove_dir_all(&dir); + // precisely typed and still declines: `except*` has no lowering let source = "\ -def holder(label: str) -> ((str) -> str): - current = label - def swap(next: str) -> str: - nonlocal current - previous = current - current = next - return previous - return swap +def precise(a: int) -> int: + return a + 1 + +def grouped(a: int) -> int: + out = a + try: + pass + except* ValueError: + out = 0 + return out "; - if build_source( + // `--no-any` is satisfied: nothing here is gradual + match build_source( source, - "by_diff_cellleak", + "by_diff_reqnative", &toolchain, &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); + &Options { + no_any: true, + ..Options::default() + }, + ) { + Ok(built) => assert_eq!(built.declined.len(), 1, "{:?}", built.declined), + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + } + + // `--require-native` is not + let error = build_source( + source, + "by_diff_reqnative", + &toolchain, + &dir, + &Options { + require_native: true, + ..Options::default() + }, + ) + .unwrap_err() + .to_string(); + assert!(error.contains("require-native"), "{error}"); + assert!(error.contains("`except*`"), "{error}"); +} + +#[test] +fn no_any_accepts_a_fully_typed_module() { + let Some((_, toolchain)) = environment() else { return; + }; + let dir = std::env::temp_dir().join("by_diff_noany_ok"); + let _ = std::fs::remove_dir_all(&dir); + let source = "def f(a: int, b: int) -> int:\n return a * b\n"; + match build_source( + source, + "by_diff_noany_ok", + &toolchain, + &dir, + &Options { + no_any: true, + ..Options::default() + }, + ) { + Ok(built) => assert!(built.declined.is_empty()), + Err(error) => eprintln!("skipping: no working C toolchain ({error})"), } - // the cell holds a reference and a write must release the old one. the live - // closure has to be dropped before measuring, or its own hold on the cell reads - // as a leak - let out = run( - &python, +} + +#[test] +fn optimized_output_still_agrees_with_the_interpreter() { + // copy propagation and infallibility both rewrite the IR — the differential + // legs are what say the rewrites preserved the program + agree( + "optimized", + "\ +def area(r: float) -> float: + scaled = r * r + return 3.5 * scaled + +def compose(a: float, b: float) -> float: + x = area(a) + y = area(b) + return x + y +", + &[ + "m.area(2.0)", + "m.compose(1.0, 2.0)", + "[m.compose(a, a) for a in (0.0, -1.5, 3.25)]", + ], + ); +} + +#[test] +fn a_declined_function_is_reported_with_a_reason() { + let Some((_, toolchain)) = environment() else { + return; + }; + let dir: PathBuf = std::env::temp_dir().join("by_diff_declined"); + let _ = std::fs::remove_dir_all(&dir); + + let source = "\ +def fast(a: int) -> int: + return a + 1 + +def slow(a: int) -> None: + try: + pass + except* ValueError: + pass +"; + let Ok(built) = build_source( + source, + "by_diff_declined", + &toolchain, &dir, - "import gc, sys, by_diff_cellleak as m\n\ - label = 'x' * 40\n\ - other = 'y' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 swap = m.holder(label)\n\ - \x20 swap(other)\n\ - \x20 swap(label)\n\ - del swap\n\ - print('refs', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - gc.collect()\n\ - objects = len(gc.get_objects())\n\ - for _ in range(2000):\n\ - \x20 held = m.holder(label)\n\ - \x20 held(other)\n\ - del held\n\ - gc.collect()\n\ - print('envs', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", + &Options::default(), + ) else { + eprintln!("skipping: no working C toolchain"); + return; + }; + assert_eq!(built.declined.len(), 1); + assert_eq!(built.declined[0].name, "slow"); + assert!( + built.declined[0].reason.contains("`except*`"), + "{:?}", + built.declined[0] ); - assert_eq!(out, "refs stable\nenvs stable"); } #[test] -fn generators_agree() { - agree_with_declines( - "generators", +fn field_access_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree( + "fields", "\ -def counted(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 +data class Point: + x: int + y: int -def three() -> object: - yield 1 - yield 2 - yield 3 + def total(self) -> int: + return self.x + self.y -def accumulating(words: list[str]) -> object: - seen = \"\" - for w in words: - seen = seen + w - yield seen + def shift(self, d: int) -> int: + self.x = self.x + d + return self.x -def pairs(xs: list[int], ys: list[int]) -> object: - for a in xs: - for b in ys: - yield a * b +data class Line: + a: Point + b: Point -def nothing() -> object: - if False: - yield 1 + def span(self) -> int: + return self.b.x - self.a.x -def early(n: int) -> object: - yield n - return + def retarget(self, p: Point) -> int: + self.b = p + return self.span() -def echoing() -> object: - total = 0 - while True: - got = yield total - total = total + 1 +def sum_of(p: Point) -> int: + return p.x + p.y + +def bump_twice(p: Point) -> int: + p.shift(1) + p.shift(2) + return p.x ", &[ - "list(m.counted(4))", - "list(m.three())", - "list(m.accumulating(['a', 'bb', 'ccc']))", - "list(m.pairs([1, 2, 3], [10, 20]))", - "list(m.nothing())", - "list(m.early(7))", - // arbitrary precision survives the suspension - "list(m.counted(3))[-1] + 10 ** 20", - // partial consumption, then more - "[(g := m.counted(5), next(g), next(g), list(g))[1:]]", - // it is a real iterator, so everything that takes one works - "sum(m.counted(5))", - "sorted(m.three(), reverse=True)", - "[x for x in m.counted(3) if x]", - "list(zip(m.counted(3), m.three()))", - // exhaustion keeps raising - "[type(e).__name__ for e in [_capture(next, m.nothing())]]", - "[(g := m.early(1), next(g), type(_capture(next, g)).__name__, type(_capture(next, g)).__name__)[2:]]", - // `send` is what the `yield` expression evaluates to - "[(e := m.echoing(), next(e), e.send(9), e.send(9))[1:]]", - // `close` exhausts it - "[(g := m.counted(9), next(g), g.close(), type(_capture(next, g)).__name__)[3:]]", + "m.Point(3, 4).total()", + "m.sum_of(m.Point(3, 4))", + "m.Point(3, 4).shift(10)", + "m.bump_twice(m.Point(0, 0))", + "m.Line(m.Point(1, 2), m.Point(10, 20)).span()", + "m.Line(m.Point(1, 2), m.Point(10, 20)).a.y", + "m.Line(m.Point(1, 2), m.Point(10, 20)).retarget(m.Point(100, 0))", + // a write through the python-visible setter, then a native read + "[(p := m.Point(1, 2), setattr(p, 'x', 9), p.total())[-1]]", + "[m.sum_of(m.Point(a, b)) for a in (0, -1, 10 ** 20) for b in (0, 5)]", ], ); } #[test] -fn a_generator_is_a_real_iterator_to_python() { +fn a_field_setter_checks_its_value() { + // the same documented delta as the constructor: an unboxed field cannot hold + // the wrong representation, and `@dataclass` does not check assignments let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_geniter"); + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_setcheck"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -def counted(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 +data class Point: + x: int + y: int + +frozen data class Fixed: + n: int "; if build_source( source, - "by_diff_geniter", + "by_diff_setcheck", &toolchain, &dir, &Options::default(), @@ -4001,679 +4497,2316 @@ def counted(n: int) -> object: let out = run( &python, &dir, - "import by_diff_geniter as m\n\ - g = m.counted(3)\n\ - print(iter(g) is g)\n\ - print(hasattr(g, '__next__'), hasattr(g, 'send'), hasattr(g, 'close'))\n\ - print(list(g))\n", - ); - assert_eq!(out, "True\nTrue True True\n[0, 1, 2]"); + "import by_diff_setcheck as m\n\ + p = m.Point(1, 2)\n\ + p.x = 5\n\ + print(p.x)\n\ + for attempt in (lambda: setattr(p, 'x', 'a'), lambda: delattr(p, 'x'), lambda: setattr(m.Fixed(1), 'n', 2)):\n\ + \x20 try:\n attempt()\n\ + \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ + \x20 else:\n print('accepted')\n", + ); + assert_eq!(out, "5\nTypeError\nAttributeError\nAttributeError"); } -/// the generator shapes every leak test below drives, built once -/// -/// `require_native` is what makes the answers the *compiled* generator's: a -/// declined function would run from its interpreted definition and leak nothing, -/// so the test would pass without exercising anything -fn leak_module(tag: &'static str) -> Option<(String, std::path::PathBuf)> { - let (python, toolchain) = environment()?; - let dir = std::env::temp_dir().join(tag); +#[test] +fn a_class_typed_argument_is_checked_at_the_boundary() { + // a compiled parameter typed as a native class is a pointer to its struct, + // so python handing over anything else has to be caught here — the + // alternative is a wild pointer dereference + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_argcheck"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Boom(Exception): - pass - -def repeat(label: str, times: int) -> object: - i = 0 - while i < times: - yield label - i = i + 1 - -def guarded(times: int) -> object: - i = 0 - while i < times: - try: - yield i - except Boom: - yield -1 - i = i + 1 +data class Point: + x: int -def in_handler(times: int) -> object: - i = 0 - while i < times: - try: - raise Boom() - except Boom: - yield i - i = i + 1 +def read(p: Point) -> int: + return p.x "; - let options = Options { - require_native: true, - ..Options::default() - }; - if build_source(source, tag, &toolchain, &dir, &options).is_err() { + if build_source( + source, + "by_diff_argcheck", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { eprintln!("skipping: no working C toolchain"); - return None; + return; } - Some((python, dir)) + let out = run( + &python, + &dir, + "import by_diff_argcheck as m\n\ + print(m.read(m.Point(7)))\n\ + for bad in ('a', 1, None):\n\ + \x20 try:\n m.read(bad)\n\ + \x20 except TypeError as e:\n print(e)\n\ + \x20 else:\n print('accepted')\n", + ); + assert_eq!( + out, + "7\nexpected by_diff_argcheck.Point, got str\n\ + expected by_diff_argcheck.Point, got int\n\ + expected by_diff_argcheck.Point, got NoneType" + ); } -/// the two counters every leak test here uses -/// -/// a *total* object count is too noisy to see one leaked object per iteration -/// until the iteration count is large, and a count of some unrelated object is -/// blind to it entirely — so each of these watches the object that would actually -/// be leaked: instances of one class, and the reference count of one instance -const LEAK_INSTRUMENTS: &str = "\ -import gc, sys -def live(kind): - gc.collect() - return sum(1 for o in gc.get_objects() if type(o) is kind) -def leaked(kind, once): - base = live(kind) - for _ in range(50): - once() - return live(kind) - base -"; - #[test] -fn a_generator_does_not_leak_its_state() { - let Some((python, dir)) = leak_module("by_diff_genleak") else { +fn a_field_read_does_not_leak() { + let Some((python, toolchain)) = environment() else { return; }; - // the state object holds every parameter, and dropping it must release them — - // including when the generator is abandoned part-way through. - // - // watching the *label* alone is what let a leaked `GeneratorExit` through for - // as long as it did: the parameter is released either way, so the count below - // it is the one that moves + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_fieldleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + +def label_of(h: Holder) -> str: + return h.label + +data class Nest: + inner: Holder + +def inner_label(n: Nest) -> str: + return n.inner.label +"; + if build_source( + source, + "by_diff_fieldleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // a field read hands back a *borrowed* reference, so the register that takes + // it must retain — and release. `str` is not gc-tracked, so this measures the + // refcount rather than the object count let out = run( &python, &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 list(m.repeat(label, 3))\n\ - print('drained', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.repeat(label, 9)\n\ - \x20 next(g)\n\ - del g\n\ - gc.collect()\n\ - print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - def abandon():\n\ - \x20 g = m.repeat(label, 9)\n\ - \x20 next(g)\n\ - print('exit objects', leaked(GeneratorExit, abandon))\n" - ), + "import sys, by_diff_fieldleak as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + n = m.Nest(h)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.label_of(h)\n\ + \x20 m.inner_label(n)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n", ); - assert_eq!(out, "drained stable\nabandoned stable\nexit objects 0"); + assert_eq!(out, "stable"); } -/// finalising a suspended generator throws `GeneratorExit` in, and the unwind has -/// to release it — the frame does not own what it hands to the error state #[test] -fn ending_a_generator_releases_the_generator_exit() { - let Some((python, dir)) = leak_module("by_diff_genexit") else { +fn a_constructor_result_is_used_natively() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); return; - }; - let out = run( - &python, - &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genexit as m\n\ - def abandoned():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - def exhausted():\n\ - \x20 list(m.repeat('x', 3))\n\ - def closed():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - \x20 g.close()\n\ - def in_handler():\n\ - \x20 g = m.in_handler(9)\n\ - \x20 next(g)\n\ - print('abandoned', leaked(GeneratorExit, abandoned))\n\ - print('exhausted', leaked(GeneratorExit, exhausted))\n\ - print('closed', leaked(GeneratorExit, closed))\n\ - print('in handler', leaked(GeneratorExit, in_handler))\n" - ), + } + agree( + "ctornative", + "\ +data class Point: + x: int + y: int + + def total(self) -> int: + return self.x + self.y + +data class Line: + a: Point + b: Point + +def diag(a: int) -> int: + return Point(a, a).x + +def build(a: int, b: int) -> int: + return Line(Point(a, b), Point(b, a)).b.x + +def widened(a: int) -> object: + return Point(a, a) + +def chained(a: int) -> int: + p = Point(a, a + 1) + q = Point(p.y, p.x) + return q.total() + p.total() +", + &[ + "m.diag(7)", + "m.build(3, 9)", + "m.widened(4).total()", + "m.chained(5)", + "[m.chained(a) for a in (0, -3, 10 ** 20)]", + ], + ); +} + +#[test] +fn a_declined_callee_does_not_break_the_build() { + // the whole module used to fail to compile: the emitted call named a symbol + // that was never defined + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree_with_declines( + "declinechain", + "\ +def helper[T](a: T) -> T: + return a + +def caller(a: int) -> int: + return helper(a) + a + +data class Point: + x: int + + def bad[T](self, a: T) -> int: + return self.x + +def read(p: Point) -> int: + return p.x + +def alone(a: int) -> int: + return a + 1 +", + &[ + "m.helper(1)", + "m.caller(2)", + "m.read(m.Point(5))", + "m.Point(5).bad(1)", + "m.alone(3)", + ], + ); +} + +#[test] +fn direct_method_dispatch_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree( + "direct", + "\ +data class Point: + x: int + y: int + + def total(self) -> int: + return self.x + self.y + + def scaled(self, k: int) -> int: + return self.total() * k + + def shifted(self, d: int) -> int: + self.x = self.x + d + return self.total() + + def label(self) -> str: + return \"p\" + +data class Line: + a: Point + b: Point + + def span(self) -> int: + return self.b.total() - self.a.total() + + def widest(self, other: Line) -> int: + mine = self.span() + theirs = other.span() + if mine > theirs: + return mine + return theirs + +def drive(a: int, b: int) -> int: + p = Point(a, b) + return p.scaled(2) + p.total() + p.shifted(1) + +def countdown(p: Point, n: int) -> int: + total = 0 + for _ in range(n): + total = total + p.total() + return total +", + &[ + "m.Point(3, 4).scaled(5)", + "m.Point(3, 4).label()", + "m.drive(1, 2)", + "m.countdown(m.Point(2, 3), 4)", + "m.Line(m.Point(1, 1), m.Point(10, 10)).span()", + "m.Line(m.Point(1, 1), m.Point(10, 10)).widest(m.Line(m.Point(0, 0), m.Point(2, 2)))", + "[m.drive(a, a) for a in (0, -7, 10 ** 20)]", + // a python caller reaches the same method through the type object + "[getattr(m.Point(3, 4), 'scaled')(2)]", + ], + ); +} + +#[test] +fn a_direct_method_call_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_methleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + + def get(self) -> str: + return self.label + +def twice(h: Holder) -> str: + return h.get() + h.get() +"; + if build_source( + source, + "by_diff_methleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_methleak as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.twice(h)\n\ + \x20 h.get()\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n", + ); + assert_eq!(out, "stable"); +} + +#[test] +fn a_borrowed_intermediate_does_not_leak_or_lose_a_reference() { + // the borrow pass drops the retain/release pair around `n.inner`. getting it + // wrong is either a leak or a use-after-free, so both directions are checked: + // the label's refcount must be *stable*, and the inner holder must survive + // being dropped from python while a compiled read is in flight + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_borrow"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Holder: + label: str + +data class Nest: + inner: Holder + +def inner_label(n: Nest) -> str: + return n.inner.label +"; + if build_source( + source, + "by_diff_borrow", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_borrow as m\n\ + label = 'x' * 40\n\ + h = m.Holder(label)\n\ + n = m.Nest(h)\n\ + holder_refs = sys.getrefcount(h)\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.inner_label(n)\n\ + print('label', 'stable' if sys.getrefcount(label) == before else 'moved')\n\ + print('holder', 'stable' if sys.getrefcount(h) == holder_refs else 'moved')\n\ + del h\n\ + print(m.inner_label(n)[:4])\n", + ); + assert_eq!(out, "label stable\nholder stable\nxxxx"); +} + +#[test] +fn calling_a_callable_value_agrees() { + agree( + "callvalue", + "\ +def apply(f: object, a: int) -> object: + return f(a) + +def apply2(f: object, a: int, b: int) -> object: + return f(a, b) + +def indirect(a: int) -> object: + fn = abs + return fn(a) + +def shadowed(len: object, s: str) -> object: + return len(s) + +def nothing(f: object) -> object: + return f() +", + &[ + "m.apply(abs, -5)", + "m.apply(str, 5)", + "m.apply2(max, 3, 9)", + "m.indirect(-7)", + // a parameter shadowing a builtin has to win + "m.shadowed(lambda s: 'shadowed', 'abc')", + "m.nothing(dict)", + // and a value that is not callable raises the same way + "[type(e).__name__ for e in [_capture(m.apply, 3, 4)]]", + "[type(e).__name__ for e in [_capture(m.nothing, None)]]", + ], + ); +} + +#[test] +fn a_borrow_survives_a_finalizer_that_runs_a_collection() { + // the borrow pass may only skip the retain where nothing can run in between. + // a `__del__` firing during the window would be the way to catch it wrong + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 10)) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + let dir = std::env::temp_dir().join("by_diff_finalizer"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +data class Inner: + label: str + +data class Outer: + inner: Inner + +def read_then_call(o: Outer, f: object) -> str: + held = o.inner.label + f() + return held + o.inner.label +"; + if build_source( + source, + "by_diff_finalizer", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import gc, by_diff_finalizer as m\n\ + class Noisy:\n\ + \x20 def __del__(self):\n gc.collect()\n\ + o = m.Outer(m.Inner('alpha'))\n\ + def churn():\n\ + \x20 Noisy()\n\ + \x20 o.inner = m.Inner('bravo')\n\ + for _ in range(5000):\n\ + \x20 m.read_then_call(o, churn)\n\ + print(m.read_then_call(o, lambda: None))\n", + ); + assert_eq!(out, "bravobravo"); +} + +#[test] +fn reading_a_global_as_a_value_agrees() { + agree( + "globalread", + "\ +LIMIT = 10 + +def limit() -> object: + return LIMIT + +def builtin_alias(a: int) -> object: + fn = abs + return fn(a) + +def missing() -> object: + return not_defined_anywhere +", + &[ + "m.limit()", + "m.builtin_alias(-9)", + "[type(e).__name__ for e in [_capture(m.missing)]]", + // a rebound global is observed, because the read is not cached + "[(setattr(m, 'LIMIT', 99), m.limit())[-1]]", + ], + ); +} + +#[test] +fn closures_agree() { + agree_with_declines( + "closures", + "\ +def make_adder(n: int) -> object: + def add(a: int) -> int: + return a + n + return add + +def make_pair(a: int, b: str) -> object: + def describe(times: int) -> str: + return b * times + str(a) + return describe + +def helper(a: int) -> int: + def double(x: int) -> int: + return x * 2 + return double(a) + double(a) + +def compose(f: object, g: object) -> object: + def both(n: int) -> object: + return f(g(n)) + return both + +def counted(n: int) -> int: + def step(a: int) -> int: + return a + n + total = 0 + for i in range(4): + total = step(total) + return total + +def used_early(a: int) -> int: + if a > 0: + return later(a) + def later(x: int) -> int: + return x + return later(a) +", + &[ + "m.make_adder(5)(3)", + // two closures from one function must have independent environments + "[(m.make_adder(1), m.make_adder(100))[0](0), m.make_adder(100)(0)]", + "m.make_pair(7, 'ab')(2)", + "m.helper(4)", + "m.compose(abs, lambda n: n - 10)(3)", + "m.counted(2)", + "[m.make_adder(a)(a) for a in (0, -3, 10 ** 20)]", + // the closure is a real callable python can inspect and pass around + "callable(m.make_adder(1))", + "sorted([3, 1, 2], key=m.make_adder(0))", + // and a wrong argument count still raises + "[type(e).__name__ for e in [_capture(m.make_adder(1))]]", + // python raises here, and the interpreted fallback is what reports it + "[type(e).__name__ for e in [_capture(m.used_early, 5)]]", + ], + ); +} + +#[test] +fn a_closure_does_not_leak_its_environment() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_closureleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def make(label: str) -> object: + def get(times: int) -> str: + return label * times + return get +"; + if build_source( + source, + "by_diff_closureleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the environment holds the captured `str`, and releasing the closure has to + // release the environment, which releases the field + let out = run( + &python, + &dir, + "import sys, by_diff_closureleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.make(label)(1)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n\ + held = m.make(label)\n\ + print(sys.getrefcount(label) > before)\n\ + del held\n\ + print(sys.getrefcount(label) == before)\n", + ); + assert_eq!(out, "stable\nTrue\nTrue"); +} + +#[test] +fn a_closure_environment_is_not_visible_in_the_module() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_envhidden"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def make(n: int) -> object: + def get() -> int: + return n + return get +"; + if build_source( + source, + "by_diff_envhidden", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_envhidden as m\n\ + print([n for n in dir(m) if 'env' in n])\n\ + print(m.make(3)())\n", + ); + assert_eq!(out, "[]\n3"); +} + +#[test] +fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { + // the exception edge is a CFG edge, and the refcount pass used not to follow it + // — so everything the `try` body had written leaked on the exceptional path + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_handlerleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def guarded(words: list[str], index: int) -> str: + held = \"held\" + words[0] + try: + return held + words[index] + except IndexError: + return held +"; + if build_source( + source, + "by_diff_handlerleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import sys, by_diff_handlerleak as m\n\ + words = ['x' * 40]\n\ + label = words[0]\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 m.guarded(words, 9)\n\ + after = sys.getrefcount(label)\n\ + print('stable' if after == before else f'leaked {before}->{after}')\n\ + print(m.guarded(words, 0)[:8])\n", + ); + assert_eq!(out, "stable\nheldxxxx"); +} + +#[test] +fn a_mutable_capture_agrees() { + // python closes over the *variable*, so all of these depend on both frames seeing + // one cell — a copy at `def` time would give different answers for every one + agree( + "cells", + "\ +def counter() -> (() -> int): + n = 0 + def get() -> int: + return n + n = 1 + return get + +def bumper() -> (() -> int): + n = 0 + def bump() -> int: + nonlocal n + n = n + 1 + return n + return bump + +def loop_closures() -> list[object]: + out = [] + i = 0 + while i < 3: + def show() -> int: + return i + out.append(show) + i = i + 1 + return out + +def shared_pair(start: int) -> list[object]: + def read() -> int: + return start + def write(v: int) -> int: + nonlocal start + start = v + return start + return [read, write] + +def accumulate(values: list[int]) -> int: + total = 0 + def add(v: int) -> int: + nonlocal total + total = total + v + return total + for v in values: + add(v) + return total +", + &[ + "m.counter()()", + "[(b := m.bumper(), b(), b(), b())[1:]]", + "[f() for f in m.loop_closures()]", + // one cell: the write through one closure is visible through the other + "[(p := m.shared_pair(5), p[0](), p[1](9), p[0]())[1:]]", + "m.accumulate([1, 2, 3])", + "[m.accumulate([a, a]) for a in (0, -4, 10 ** 20)]", + ], + ); +} + +#[test] +fn reading_a_cell_before_it_is_written_raises_the_way_python_does() { + // a cell starts unset, and NULL has to read back as an error rather than a zero + agree_with_declines( + "cellunset", + "\ +def early() -> object: + def get() -> int: + return n + out = get + n = 1 + return out + +def early_call() -> object: + def get() -> int: + return n + result = _capture_local(get) + n = 1 + return result + +def _capture_local(f: object) -> object: + try: + return f() + except NameError as e: + return type(e).__name__ +", + &[ + // reading it after the write is fine + "m.early()()", + // reading it before is `UnboundLocalError`, which is a `NameError` + "m.early_call()", + ], + ); +} + +#[test] +fn a_shared_cell_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_cellleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def holder(label: str) -> ((str) -> str): + current = label + def swap(next: str) -> str: + nonlocal current + previous = current + current = next + return previous + return swap +"; + if build_source( + source, + "by_diff_cellleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the cell holds a reference and a write must release the old one. the live + // closure has to be dropped before measuring, or its own hold on the cell reads + // as a leak + let out = run( + &python, + &dir, + "import gc, sys, by_diff_cellleak as m\n\ + label = 'x' * 40\n\ + other = 'y' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 swap = m.holder(label)\n\ + \x20 swap(other)\n\ + \x20 swap(label)\n\ + del swap\n\ + print('refs', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + gc.collect()\n\ + objects = len(gc.get_objects())\n\ + for _ in range(2000):\n\ + \x20 held = m.holder(label)\n\ + \x20 held(other)\n\ + del held\n\ + gc.collect()\n\ + print('envs', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", + ); + assert_eq!(out, "refs stable\nenvs stable"); +} + +#[test] +fn generators_agree() { + agree_with_declines( + "generators", + "\ +def counted(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 + +def three() -> object: + yield 1 + yield 2 + yield 3 + +def accumulating(words: list[str]) -> object: + seen = \"\" + for w in words: + seen = seen + w + yield seen + +def pairs(xs: list[int], ys: list[int]) -> object: + for a in xs: + for b in ys: + yield a * b + +def nothing() -> object: + if False: + yield 1 + +def early(n: int) -> object: + yield n + return + +def echoing() -> object: + total = 0 + while True: + got = yield total + total = total + 1 +", + &[ + "list(m.counted(4))", + "list(m.three())", + "list(m.accumulating(['a', 'bb', 'ccc']))", + "list(m.pairs([1, 2, 3], [10, 20]))", + "list(m.nothing())", + "list(m.early(7))", + // arbitrary precision survives the suspension + "list(m.counted(3))[-1] + 10 ** 20", + // partial consumption, then more + "[(g := m.counted(5), next(g), next(g), list(g))[1:]]", + // it is a real iterator, so everything that takes one works + "sum(m.counted(5))", + "sorted(m.three(), reverse=True)", + "[x for x in m.counted(3) if x]", + "list(zip(m.counted(3), m.three()))", + // exhaustion keeps raising + "[type(e).__name__ for e in [_capture(next, m.nothing())]]", + "[(g := m.early(1), next(g), type(_capture(next, g)).__name__, type(_capture(next, g)).__name__)[2:]]", + // `send` is what the `yield` expression evaluates to + "[(e := m.echoing(), next(e), e.send(9), e.send(9))[1:]]", + // `close` exhausts it + "[(g := m.counted(9), next(g), g.close(), type(_capture(next, g)).__name__)[3:]]", + ], + ); +} + +#[test] +fn a_generator_is_a_real_iterator_to_python() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_geniter"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def counted(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 +"; + if build_source( + source, + "by_diff_geniter", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + "import by_diff_geniter as m\n\ + g = m.counted(3)\n\ + print(iter(g) is g)\n\ + print(hasattr(g, '__next__'), hasattr(g, 'send'), hasattr(g, 'close'))\n\ + print(list(g))\n", + ); + assert_eq!(out, "True\nTrue True True\n[0, 1, 2]"); +} + +/// the generator shapes every leak test below drives, built once +/// +/// `require_native` is what makes the answers the *compiled* generator's: a +/// declined function would run from its interpreted definition and leak nothing, +/// so the test would pass without exercising anything +fn leak_module(tag: &'static str) -> Option<(String, std::path::PathBuf)> { + let (python, toolchain) = environment()?; + let dir = std::env::temp_dir().join(tag); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Boom(Exception): + pass + +def repeat(label: str, times: int) -> object: + i = 0 + while i < times: + yield label + i = i + 1 + +def guarded(times: int) -> object: + i = 0 + while i < times: + try: + yield i + except Boom: + yield -1 + i = i + 1 + +def in_handler(times: int) -> object: + i = 0 + while i < times: + try: + raise Boom() + except Boom: + yield i + i = i + 1 +"; + let options = Options { + require_native: true, + ..Options::default() + }; + if build_source(source, tag, &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return None; + } + Some((python, dir)) +} + +/// the two counters every leak test here uses +/// +/// a *total* object count is too noisy to see one leaked object per iteration +/// until the iteration count is large, and a count of some unrelated object is +/// blind to it entirely — so each of these watches the object that would actually +/// be leaked: instances of one class, and the reference count of one instance +const LEAK_INSTRUMENTS: &str = "\ +import gc, sys +def live(kind): + gc.collect() + return sum(1 for o in gc.get_objects() if type(o) is kind) +def leaked(kind, once): + base = live(kind) + for _ in range(50): + once() + return live(kind) - base +"; + +#[test] +fn a_generator_does_not_leak_its_state() { + let Some((python, dir)) = leak_module("by_diff_genleak") else { + return; + }; + // the state object holds every parameter, and dropping it must release them — + // including when the generator is abandoned part-way through. + // + // watching the *label* alone is what let a leaked `GeneratorExit` through for + // as long as it did: the parameter is released either way, so the count below + // it is the one that moves + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 list(m.repeat(label, 3))\n\ + print('drained', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.repeat(label, 9)\n\ + \x20 next(g)\n\ + del g\n\ + gc.collect()\n\ + print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + def abandon():\n\ + \x20 g = m.repeat(label, 9)\n\ + \x20 next(g)\n\ + print('exit objects', leaked(GeneratorExit, abandon))\n" + ), + ); + assert_eq!(out, "drained stable\nabandoned stable\nexit objects 0"); +} + +/// finalising a suspended generator throws `GeneratorExit` in, and the unwind has +/// to release it — the frame does not own what it hands to the error state +#[test] +fn ending_a_generator_releases_the_generator_exit() { + let Some((python, dir)) = leak_module("by_diff_genexit") else { + return; + }; + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genexit as m\n\ + def abandoned():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + def exhausted():\n\ + \x20 list(m.repeat('x', 3))\n\ + def closed():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + \x20 g.close()\n\ + def in_handler():\n\ + \x20 g = m.in_handler(9)\n\ + \x20 next(g)\n\ + print('abandoned', leaked(GeneratorExit, abandoned))\n\ + print('exhausted', leaked(GeneratorExit, exhausted))\n\ + print('closed', leaked(GeneratorExit, closed))\n\ + print('in handler', leaked(GeneratorExit, in_handler))\n" + ), + ); + assert_eq!(out, "abandoned 0\nexhausted 0\nclosed 0\nin handler 0"); +} + +/// a `throw` raises at the suspension point, and whether a handler catches it or +/// it comes back out, nothing may hold a reference afterwards +#[test] +fn throwing_into_a_generator_releases_the_exception() { + let Some((python, dir)) = leak_module("by_diff_genthrow") else { + return; + }; + // two instruments over the same shapes: one instance thrown again and again, + // whose own reference count moves even though the object stays reachable, and a + // fresh instance per call, which a retained reference keeps alive as a countable + // object. + // + // the exception is a global rather than a parameter on purpose. a caught + // exception's traceback holds the frames it passed through, so a local naming it + // is a reference the *test* keeps — 50 of them, in both builds alike + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_genthrow as m\n\ + def caught():\n\ + \x20 g = m.guarded(9)\n\ + \x20 next(g)\n\ + \x20 g.throw(thrown)\n\ + def uncaught():\n\ + \x20 g = m.repeat('x', 9)\n\ + \x20 next(g)\n\ + \x20 try:\n\ + \x20 g.throw(thrown)\n\ + \x20 except m.Boom:\n\ + \x20 pass\n\ + def in_handler():\n\ + \x20 g = m.in_handler(9)\n\ + \x20 next(g)\n\ + def fresh(once):\n\ + \x20 def run():\n\ + \x20 global thrown\n\ + \x20 thrown = m.Boom()\n\ + \x20 once()\n\ + \x20 thrown = None\n\ + \x20 return run\n\ + thrown = m.Boom()\n\ + boom = thrown\n\ + gc.collect()\n\ + held = sys.getrefcount(boom)\n\ + for _ in range(50):\n\ + \x20 caught()\n\ + \x20 uncaught()\n\ + gc.collect()\n\ + print('thrown refs', sys.getrefcount(boom) - held)\n\ + print('caught', leaked(m.Boom, fresh(caught)))\n\ + print('uncaught', leaked(m.Boom, fresh(uncaught)))\n\ + print('raised inside', leaked(m.Boom, in_handler))\n" + ), + ); + assert_eq!(out, "thrown refs 0\ncaught 0\nuncaught 0\nraised inside 0"); +} + +/// the same rule outside a generator, which is where it is stated: a re-raise +/// hands the exception to the interpreter and keeps nothing +#[test] +fn a_re_raise_does_not_retain_the_exception() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_reraiseleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Boom(Exception): + pass + +def bare(x: int) -> int: + try: + raise Boom() + except Boom: + raise + +def unmatched(x: int) -> int: + try: + raise Boom() + except ValueError: + return 1 + return 2 + +def through_finally(x: int) -> int: + try: + raise Boom() + finally: + x = x + 1 +"; + let options = Options { + require_native: true, + ..Options::default() + }; + if build_source(source, "by_diff_reraiseleak", &toolchain, &dir, &options).is_err() { + eprintln!("skipping: no working C toolchain"); + return; + } + let out = run( + &python, + &dir, + &format!( + "{LEAK_INSTRUMENTS}\ + import by_diff_reraiseleak as m\n\ + def raising(fn):\n\ + \x20 def once():\n\ + \x20 try:\n\ + \x20 fn(1)\n\ + \x20 except m.Boom:\n\ + \x20 pass\n\ + \x20 return once\n\ + for name in ('bare', 'unmatched', 'through_finally'):\n\ + \x20 print(name, leaked(m.Boom, raising(getattr(m, name))))\n" + ), + ); + assert_eq!(out, "bare 0\nunmatched 0\nthrough_finally 0"); +} + +#[test] +fn delegation_agrees() { + agree_with_declines( + "delegation", + "\ +def inner(n: int) -> object: + i = 0 + while i < n: + yield i + i = i + 1 + return n * 100 + +def outer(n: int) -> object: + got = yield from inner(n) + yield got + +def chained(xs: list[int]) -> object: + yield from xs + yield from xs + +def nested(n: int) -> object: + yield from outer(n) +", + &[ + "list(m.inner(3))", + // the delegated `return` value is what the expression evaluates to + "list(m.outer(3))", + "list(m.chained([1, 2]))", + "list(m.nested(2))", + "list(m.chained([]))", + "sum(m.outer(4))", + // and the inner generator's own `StopIteration` value reaches the consumer + "[(g := m.inner(1), next(g), _capture(next, g).value)[2:]]", + ], + ); +} + +#[test] +fn async_await_agrees() { + agree_with_declines( + "asyncawait", + "\ +async def plain(n: int) -> int: + return n * 2 + +async def chained(n: int) -> int: + a = await plain(n) + b = await plain(a) + return b + 1 + +async def nothing() -> None: + pass + +async def forwards(n: int) -> int: + return await chained(n) +", + &[ + "__import__('asyncio').run(m.plain(5))", + "__import__('asyncio').run(m.chained(3))", + "__import__('asyncio').run(m.nothing())", + "__import__('asyncio').run(m.forwards(1))", + "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", + ], + ); +} + +#[test] +fn the_await_protocol_agrees() { + // an `await` reaches its object through `__await__` and drives what comes back + // by sending into it. both halves have edges worth pinning: what counts as + // awaitable at all, and what a finished delegation's value *is* + agree( + "awaitproto", + "\ +async def awaited(x: object) -> object: + return await x + +async def twice(x: object, y: object) -> object: + a = await x + b = await y + return (a, b) + +async def guarded(x: object) -> object: + try: + return await x + except ValueError as e: + return 'caught ' + str(e) + +def delegated(xs: object) -> object: + got = yield from xs + yield got + +def returns(v: object) -> object: + return v + yield + +async def returns_async(v: object) -> object: + return v + +def redelegated(v: object) -> object: + got = yield from returns(v) + yield got +", + &[ + // the *other* half of the same rule: what a compiled frame's own return + // value becomes on the way out. `StopIteration` reads its argument as an + // argument *list*, so a tuple would arrive spread and an exception + // instance would be raised in place of the `StopIteration` itself + "[_value(m.returns(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2], 'ab')]", + "[_run(m.returns_async(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2])]", + "[list(m.redelegated(v)) for v in ((1, 2), (), (1,), 5, None)]", + "[_run(m.awaited(m.returns_async(v))) for v in ((1, 2), (), 5, None)]", + "_value(m.returns(ValueError('x')))", + "_value(m.returns(StopIteration(9)))", + "_run(m.returns_async(StopIteration(9)))", + // and the exception it rides on is shaped the way python shapes it + "[(lambda e: (repr(e), e.args))(_capture(next, m.returns(v))) \ + for v in ((1, 2), (), 5, None)]", + // an await that completes without ever suspending — the common case + "_run(m.awaited(_Ready(7)))", + "_run(m.twice(_Ready(1), _Ready(2)))", + // one that really suspends, driven by a loop that has to resume it + "_run(m.awaited(_sleeps(9)))", + "_run(m.awaited(_Suspends(11)))", + "_run(m.twice(_Ready(1), _sleeps(2)))", + "_run(m.twice(_Suspends(1), _Suspends(2)))", + // a bare `StopIteration` carries `None`, and a subclass carries its own + "_run(m.awaited(_Raises(StopIteration())))", + "_run(m.awaited(_Raises(StopIteration(3))))", + "_run(m.awaited(_Raises(_Sub(5))))", + // python reads the exception's field, not its `value` attribute + "_run(m.awaited(_Raises(_Shadowed(7))))", + "_Shadowed(7).value", + // a raised *type* rather than an instance still has to become one + "_run(m.awaited(_Raises(StopIteration)))", + // normalising a tuple argument would take only its first element + "_run(m.awaited(_Raises(StopIteration((1, 2)))))", + // anything else escaping the awaited object is not a return value + "_capture_async(m.awaited, _throws(ValueError))", + "_capture_async(m.awaited, _throws(KeyError))", + "_run(m.guarded(_throws(ValueError)))", + "_capture_async(m.guarded, _throws(KeyError))", + // and what is not awaitable at all says so, about the right object + "_capture_async(m.awaited, 5)", + "_capture_async(m.awaited, _counting(1))", + "_capture_async(m.awaited, _NotIter())", + "_capture_async(m.awaited, _CoroAwait())", + // `yield from` is the same machine over an iterator rather than an + // awaitable, and an exhausted one returns `None` rather than nothing + "list(m.delegated(iter([1, 2, 3])))", + "list(m.delegated(_counting(2)))", + "list(m.delegated(iter([])))", + "_capture(list, m.delegated(_RaiseIter(_Sub(5))))", + "_sent(m.delegated(_counting(2)), (7, 8))", + ], + ); +} + +#[test] +fn a_value_live_across_a_suspension_agrees() { + // python evaluates left to right, so `total + await step(i)` has the read of + // `total` on the stack while the `await` suspends. every shape here holds + // something across a suspension that has no name of its own, and `agree` is what + // says they compile rather than falling back + agree( + "parked", + "\ +async def stepped(i: int) -> int: + return (i * 7) % 13 + +async def summed(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await stepped(i) + i = i + 1 + return total + +async def paired(n: int) -> int: + return (await stepped(n)) + (await stepped(n + 1)) + +async def tripled(n: int) -> int: + return (await stepped(n)) * 100 + (await stepped(n + 1)) * 10 + (await stepped(n + 2)) + +def mixed(a: int, b: int, c: int) -> int: + return a * 100 + b * 10 + c + +async def spanning(n: int) -> int: + return mixed(n * 5, await stepped(n), await stepped(n + 1)) + +async def awaited(step: object, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await step(i) + i = i + 1 + return total + +def echoing(n: int) -> object: + total = 0 + i = 0 + while i < n: + total = total + (yield i) + i = i + 1 + return total + +def twinned() -> object: + return (yield 1) + (yield 2) + +def straddling(n: int) -> object: + yield mixed(n * 5, (yield 1), (yield 2)) + +def recovering(n: int) -> object: + total = n * 2 + try: + yield total + except ValueError: + yield total + 1 + yield total + 2 + +def crossed(xs: list[int], ys: list[int]) -> object: + for a in xs: + base = a * 100 + for b in ys: + yield base + (yield b) + +def counting(n: int) -> object: + yield n + return n * 100 + +def relayed(n: int) -> object: + base = n + 1 + got = yield from counting(n) + yield base + got +", + &[ + "__import__('asyncio').run(m.summed(40))", + "__import__('asyncio').run(m.paired(3))", + "__import__('asyncio').run(m.tripled(3))", + "__import__('asyncio').run(m.spanning(3))", + // `n * 5` is read before the first suspension and used after the *second*, + // so nothing between the two reads it. the static flow calls it dead at the + // first — a suspension is a `return` and goes nowhere — and one field has to + // carry it across both + "_sent(m.straddling(3), (4, 5))", + // arbitrary precision survives the park, which a machine word would not + "__import__('asyncio').run(m.summed(3)) + 10 ** 30", + // awaiting something that *does* suspend, so the loop resumes the frame + "__import__('asyncio').run(m.awaited(_slow, 4))", + "_sent(m.echoing(4), (10, 20, 30, 40))", + "_sent(m.twinned(), (5, 7))", + "list(m.recovering(3))", + // a `throw` the handler catches, and the parked value read after it + "_recovered(m.recovering(3), ValueError, 2)", + "_sent(m.crossed([1, 2], [7, 8]), (100, 200, 300, 400, 500, 600, 700, 800))", + "list(m.relayed(4))", + // a coroutine driven by hand, with no loop underneath it + "[(c := m.summed(3), _capture(c.send, None).value)[1:]]", + ], + ); +} + +#[test] +fn a_handled_throw_leaves_the_generator_usable() { + // the field carrying what `throw` wants raised starts null, and is *emptied* by + // writing `None` into it because no operation stores a null. reading it back as a + // second exception raised `SystemError` at every later resumption + agree( + "rethrown", + "\ +def recovering(n: int) -> object: + try: + yield n + except ValueError: + pass + yield n + 2 + yield n + 3 +", + &[ + "_recovered(m.recovering(3), ValueError, 2)", + "list(m.recovering(3))", + ], + ); +} + +#[test] +fn a_parked_value_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_parkleak"); + let _ = std::fs::remove_dir_all(&dir); + // `label + (yield i)` holds the label across the suspension, so the state object + // owns it for as long as the frame is parked + let source = "\ +def tagged(label: str, times: int) -> object: + i = 0 + while i < times: + yield label + (yield i) + i = i + 1 + +def handling(label: str) -> object: + try: + yield label + except ValueError: + yield label + \"!\" + yield label + \"?\" +"; + if build_source( + source, + "by_diff_parkleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // driven to exhaustion, and abandoned *while suspended* with the value parked + let out = run( + &python, + &dir, + "import gc, sys, by_diff_parkleak as m\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(20000):\n\ + \x20 g = m.tagged(label, 2)\n\ + \x20 next(g)\n\ + \x20 g.send('a')\n\ + \x20 g.close()\n\ + del g\n\ + gc.collect()\n\ + print('driven', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.tagged(label, 9)\n\ + \x20 next(g)\n\ + \x20 g.send('a')\n\ + del g\n\ + gc.collect()\n\ + print('parked', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(20000):\n\ + \x20 g = m.handling(label)\n\ + \x20 next(g)\n\ + \x20 g.throw(ValueError('boom'))\n\ + del g\n\ + gc.collect()\n\ + print('handled', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + ); + // a suspension inside `except` parks the exception the handler took over from, so + // the state object is holding one when it is dropped mid-handler + assert_eq!(out, "driven stable\nparked stable\nhandled stable"); +} + +#[test] +fn a_coroutine_is_awaitable_and_not_iterable() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_coro"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +async def plain(n: int) -> int: + return n * 2 +"; + if build_source( + source, + "by_diff_coro", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // `asyncio.iscoroutine` tests the abc, and iterating a coroutine has to be an + // error rather than quietly working + let out = run( + &python, + &dir, + "import asyncio, collections.abc as abc, by_diff_coro as m\n\ + c = m.plain(2)\n\ + print(asyncio.iscoroutine(c), isinstance(c, abc.Coroutine))\n\ + print(hasattr(c, 'send'), hasattr(c, 'throw'), hasattr(c, 'close'))\n\ + try:\n list(c)\n\ + except TypeError:\n print('not iterable')\n\ + print(asyncio.run(c))\n", + ); + assert_eq!(out, "True True\nTrue True True\nnot iterable\n4"); +} + +#[test] +fn a_coroutine_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_coroleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +async def echo(label: str) -> str: + return label +"; + if build_source( + source, + "by_diff_coroleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // including a coroutine that is created and never awaited + let out = run( + &python, + &dir, + "import asyncio, gc, sys, warnings, by_diff_coroleak as m\n\ + warnings.simplefilter('ignore')\n\ + label = 'x' * 40\n\ + before = sys.getrefcount(label)\n\ + for _ in range(5000):\n\ + \x20 asyncio.run(m.echo(label))\n\ + print('awaited', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ + for _ in range(5000):\n\ + \x20 c = m.echo(label)\n\ + del c\n\ + gc.collect()\n\ + print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + ); + assert_eq!(out, "awaited stable\nabandoned stable"); +} + +#[test] +fn with_blocks_agree() { + agree_with_declines( + "withblocks", + "\ +def counted(path: str) -> int: + total = 0 + with open(path) as f: + for line in f: + total = total + 1 + return total + +def guarded(mgr: object) -> str: + with mgr: + return \"body\" + return \"after\" + +def nested(mgr: object, other: object) -> str: + with mgr: + with other: + return \"inner\" + return \"after\" + +def suppressing(mgr: object) -> str: + with mgr: + raise ValueError(\"boom\") + return \"suppressed\" + +def propagating(mgr: object) -> str: + with mgr: + raise ValueError(\"boom\") + return \"not reached\" + +def entered(mgr: object) -> object: + with mgr as value: + return value + return None +", + &[ + // `__exit__` runs on the normal path with `(None, None, None)`, and a + // `return` from inside the body is one of those paths + "_run_recording(m)", + "m.suppressing(_Swallow())", + "[type(e).__name__ for e in [_capture(m.propagating, _Pass())]]", + "m.entered(_Value(7))", + "_run_nested(m)", + ], + ); +} + +#[test] +fn an_early_exit_runs_the_finally_it_is_leaving() { + // this was a silent wrong answer in a shipped feature: a `return` or a `break` + // inside `try` skipped the `finally` entirely + agree( + "unwind", + "\ +def early(log: list[str]) -> str: + try: + return \"body\" + finally: + log.append(\"finally\") + +def looped(log: list[str], n: int) -> str: + i = 0 + while i < n: + try: + if i == 1: + break + finally: + log.append(\"f\") + i = i + 1 + return \"done\" + +def continued(log: list[str], n: int) -> int: + total = 0 + i = 0 + while i < n: + i = i + 1 + try: + if i == 2: + continue + total = total + i + finally: + log.append(\"f\") + return total + +def layered(log: list[str]) -> str: + try: + try: + return \"deep\" + finally: + log.append(\"inner\") + finally: + log.append(\"outer\") +", + &[ + "[(log := [], m.early(log), log)[1:]]", + "[(log := [], m.looped(log, 3), log)[1:]]", + "[(log := [], m.continued(log, 4), log)[1:]]", + // innermost first, and both of them + "[(log := [], m.layered(log), log)[1:]]", + ], + ); +} + +#[test] +fn parameter_defaults_agree() { + agree_with_declines( + "defaults", + "\ +def greet(name: str, greeting: str = \"hi\", times: int = 1) -> str: + return (greeting + \" \" + name) * times + +def offset(a: int, b: int = 10) -> int: + return a + b + +def flagged(a: int, on: bool = True) -> int: + if on: + return a + return -a + +def boxed_none(a: object = None) -> object: + return a + +def boxed_int(a: object = 7) -> object: + return a + +def boxed_bool(a: object = True) -> object: + return a + +def boxed_float(a: object = 1.5) -> object: + return a + +def optional(a: int, extra: object = None) -> object: + if extra is None: + return a + return extra + +def computed(a: int, b: object = []) -> object: + return b +", + &[ + "m.greet('a')", + "m.greet('a', 'yo')", + "m.greet('a', 'yo', 2)", + "m.offset(1)", + "m.offset(1, 2)", + "(m.flagged(3), m.flagged(3, False))", + "(m.optional(1), m.optional(1, 'x'))", + // too few arguments still raises, and so does too many + "[type(e).__name__ for e in [_capture(m.offset)]]", + "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", + // a computed default declines, and the interpreted twin keeps the identity + "m.computed(1)", + // an immediate written into an *object* place has to be boxed: the + // unboxed `None` is a bare byte, and `By_NewRef` of one is a NULL the + // error check reads as a failure with no exception behind it + "m.boxed_none()", + "m.boxed_none(1)", + "m.boxed_int()", + "m.boxed_bool()", + "m.boxed_float()", + "(m.boxed_none() is None, m.boxed_bool() is True)", + ], ); - assert_eq!(out, "abandoned 0\nexhausted 0\nclosed 0\nin handler 0"); } -/// a `throw` raises at the suspension point, and whether a handler catches it or -/// it comes back out, nothing may hold a reference afterwards #[test] -fn throwing_into_a_generator_releases_the_exception() { - let Some((python, dir)) = leak_module("by_diff_genthrow") else { - return; - }; - // two instruments over the same shapes: one instance thrown again and again, - // whose own reference count moves even though the object stays reachable, and a - // fresh instance per call, which a retained reference keeps alive as a countable - // object. - // - // the exception is a global rather than a parameter on purpose. a caught - // exception's traceback holds the frames it passed through, so a local naming it - // is a reference the *test* keeps — 50 of them, in both builds alike - let out = run( - &python, - &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_genthrow as m\n\ - def caught():\n\ - \x20 g = m.guarded(9)\n\ - \x20 next(g)\n\ - \x20 g.throw(thrown)\n\ - def uncaught():\n\ - \x20 g = m.repeat('x', 9)\n\ - \x20 next(g)\n\ - \x20 try:\n\ - \x20 g.throw(thrown)\n\ - \x20 except m.Boom:\n\ - \x20 pass\n\ - def in_handler():\n\ - \x20 g = m.in_handler(9)\n\ - \x20 next(g)\n\ - def fresh(once):\n\ - \x20 def run():\n\ - \x20 global thrown\n\ - \x20 thrown = m.Boom()\n\ - \x20 once()\n\ - \x20 thrown = None\n\ - \x20 return run\n\ - thrown = m.Boom()\n\ - boom = thrown\n\ - gc.collect()\n\ - held = sys.getrefcount(boom)\n\ - for _ in range(50):\n\ - \x20 caught()\n\ - \x20 uncaught()\n\ - gc.collect()\n\ - print('thrown refs', sys.getrefcount(boom) - held)\n\ - print('caught', leaked(m.Boom, fresh(caught)))\n\ - print('uncaught', leaked(m.Boom, fresh(uncaught)))\n\ - print('raised inside', leaked(m.Boom, in_handler))\n" - ), +fn a_default_filled_at_a_native_call_site_agrees() { + // `parameter_defaults_agree` calls these from python, which fills a default in the + // *wrapper* — where the value is already an object. a compiled caller fills it + // inline instead, and that path pushed the default with no coercion at all: a bare + // `length=0` reaching an unannotated parameter put a tagged integer where the + // callee declares a `PyObject *`. it was `shutil.py` failing to build that found it, + // and only because the two representations differ enough for a C compiler to object + agree( + "native_defaults", + "\ +def taking(a, b, length=0, name=\"n\", scale=1.5, on=True, extra=None): + return (a, b, length, name, scale, on, extra) + + +def calling(x: int) -> object: + return taking(x, x) + + +def partly(x: int) -> object: + return taking(x, x, 3) + + +def by_keyword(x: int) -> object: + return taking(x, x, on=False) + + +def annotated(a: int, step: int = 2) -> int: + return a + step + + +def calling_annotated(x: int) -> int: + return annotated(x) +", + &[ + "m.calling(1)", + "m.partly(2)", + "m.by_keyword(3)", + "[m.calling_annotated(n) for n in (0, 5)]", + "m.taking(1, 2)", + "(m.calling(1)[3], type(m.calling(1)[4]).__name__, m.calling(1)[6] is None)", + ], ); - assert_eq!(out, "thrown refs 0\ncaught 0\nuncaught 0\nraised inside 0"); } -/// the same rule outside a generator, which is where it is stated: a re-raise -/// hands the exception to the interpreter and keeps nothing #[test] -fn a_re_raise_does_not_retain_the_exception() { +fn a_string_default_does_not_leak() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_reraiseleak"); + let dir = std::env::temp_dir().join("by_diff_defaultleak"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Boom(Exception): - pass - -def bare(x: int) -> int: - try: - raise Boom() - except Boom: - raise - -def unmatched(x: int) -> int: - try: - raise Boom() - except ValueError: - return 1 - return 2 - -def through_finally(x: int) -> int: - try: - raise Boom() - finally: - x = x + 1 +def padded(a: str, fill: str = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\") -> str: + return a + fill "; - let options = Options { - require_native: true, - ..Options::default() - }; - if build_source(source, "by_diff_reraiseleak", &toolchain, &dir, &options).is_err() { + if build_source( + source, + "by_diff_defaultleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { eprintln!("skipping: no working C toolchain"); return; } + // the wrapper releases its arguments, so a default handed over borrowed would be + // released twice — and one handed over with an extra reference would leak let out = run( &python, &dir, - &format!( - "{LEAK_INSTRUMENTS}\ - import by_diff_reraiseleak as m\n\ - def raising(fn):\n\ - \x20 def once():\n\ - \x20 try:\n\ - \x20 fn(1)\n\ - \x20 except m.Boom:\n\ - \x20 pass\n\ - \x20 return once\n\ - for name in ('bare', 'unmatched', 'through_finally'):\n\ - \x20 print(name, leaked(m.Boom, raising(getattr(m, name))))\n" - ), + "import sys, by_diff_defaultleak as m\n\ + fill = 'x' * 40\n\ + before = sys.getrefcount(fill)\n\ + for _ in range(20000):\n\ + \x20 m.padded('a')\n\ + print('stable' if sys.getrefcount(fill) == before else 'moved')\n\ + print(m.padded('a')[:3])\n", ); - assert_eq!(out, "bare 0\nunmatched 0\nthrough_finally 0"); + assert_eq!(out, "stable\naxx"); } #[test] -fn delegation_agrees() { +fn keyword_arguments_agree() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } agree_with_declines( - "delegation", + "keywords", "\ -def inner(n: int) -> object: - i = 0 - while i < n: - yield i - i = i + 1 - return n * 100 +def offset(a: int, b: int = 10) -> int: + return a + b + +def described(name: str, sep: str = \"-\", times: int = 1) -> str: + return (name + sep) * times + +def caller(a: int) -> int: + return offset(a, b=100) + offset(b=1, a=a) + offset(a) + +data class Point: + x: int + y: int + + def shifted(self, dx: int = 0, dy: int = 0) -> int: + return self.x + dx + self.y + dy +", + &[ + // a python caller, by name and by position + "m.offset(1, b=2)", + "m.offset(b=2, a=1)", + "m.offset(1)", + "m.described('a', times=3)", + "m.described(name='a', sep='+', times=2)", + // a compiled caller resolves the names against the callee's signature + "m.caller(1)", + // and a method's keywords work through the type object + "m.Point(1, 2).shifted(dy=10)", + "m.Point(1, 2).shifted(3, dy=10)", + // the error cases match + "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", + "[str(_capture_kw(m.offset, (1,), {'a': 2}))]", + "[str(_capture_kw(m.offset, (), {'zzz': 1}))]", + "[str(_capture(m.offset))]", + ], + ); +} + +#[test] +fn variadic_parameters_agree() { + agree_with_declines( + "variadic", + "\ +def total(*values: int) -> int: + out = 0 + for v in values: + out = out + v + return out + +def named(prefix: str, *rest: str) -> str: + out = prefix + for r in rest: + out = out + r + return out + +def options(a: int, **rest: object) -> int: + return a + len(rest) + +def both(a: int, *rest: int, **named: object) -> int: + return a + len(rest) + len(named) + +def tupled(*values: int) -> object: + return values + +def mapped(**named: object) -> object: + return named -def outer(n: int) -> object: - got = yield from inner(n) - yield got +def calls_total(a: int, b: int) -> int: + return total(a, b, 3) -def chained(xs: list[int]) -> object: - yield from xs - yield from xs +def calls_options(a: int) -> int: + return options(a, x=1, y=2) -def nested(n: int) -> object: - yield from outer(n) +def calls_both(a: int) -> int: + return both(a, 1, 2, k=3, j=4) + +def calls_none() -> int: + return total() ", &[ - "list(m.inner(3))", - // the delegated `return` value is what the expression evaluates to - "list(m.outer(3))", - "list(m.chained([1, 2]))", - "list(m.nested(2))", - "list(m.chained([]))", - "sum(m.outer(4))", - // and the inner generator's own `StopIteration` value reaches the consumer - "[(g := m.inner(1), next(g), _capture(next, g).value)[2:]]", + "m.total()", + "m.total(1, 2, 3)", + "(m.named('a'), m.named('a', 'b', 'c'))", + "(m.options(1), m.options(1, x=2, y=3))", + "m.both(1, 2, 3, k=4)", + // the body sees a real tuple and a real dict + "m.tupled(1, 2)", + "sorted(m.mapped(b=2, a=1).items())", + "type(m.tupled()).__name__", + "type(m.mapped()).__name__", + // a keyword that names no parameter goes to `**kwargs`, or raises + "[str(_capture_kw(m.named, ('a',), {'zzz': 1}))]", + "[str(_capture(m.options))]", + "[m.total(*a) for a in ([1], [1, 2])]", + // and a *compiled* caller packs the tuple and the dict itself + "m.calls_total(1, 2)", + "m.calls_options(5)", + "m.calls_both(1)", + "m.calls_none()", ], ); } #[test] -fn async_await_agrees() { - agree_with_declines( - "asyncawait", - "\ -async def plain(n: int) -> int: - return n * 2 - -async def chained(n: int) -> int: - a = await plain(n) - b = await plain(a) - return b + 1 - -async def nothing() -> None: - pass +fn a_variadic_argument_does_not_leak() { + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_varleak"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +def joined(*parts: str) -> int: + total = 0 + for p in parts: + total = total + len(p) + return total -async def forwards(n: int) -> int: - return await chained(n) -", - &[ - "__import__('asyncio').run(m.plain(5))", - "__import__('asyncio').run(m.chained(3))", - "__import__('asyncio').run(m.nothing())", - "__import__('asyncio').run(m.forwards(1))", - "[__import__('asyncio').run(m.plain(a)) for a in (0, -3, 10 ** 20)]", - ], +def mapped(**named: object) -> int: + return len(named) +"; + if build_source( + source, + "by_diff_varleak", + &toolchain, + &dir, + &Options::default(), + ) + .is_err() + { + eprintln!("skipping: no working C toolchain"); + return; + } + // the wrapper *builds* the tuple and the dict, so it owns them and has to release + // them — and the elements it put in are borrowed from the caller + let out = run( + &python, + &dir, + "import gc, sys, by_diff_varleak as m\n\ + part = 'x' * 40\n\ + before = sys.getrefcount(part)\n\ + for _ in range(20000):\n\ + \x20 m.joined(part, part)\n\ + \x20 m.mapped(a=part)\n\ + print('refs', 'stable' if sys.getrefcount(part) == before else 'leaked')\n\ + gc.collect()\n\ + objects = len(gc.get_objects())\n\ + for _ in range(5000):\n\ + \x20 m.joined(part)\n\ + \x20 m.mapped(a=part, b=part)\n\ + gc.collect()\n\ + print('objects', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", ); + assert_eq!(out, "refs stable\nobjects stable"); } #[test] -fn the_await_protocol_agrees() { - // an `await` reaches its object through `__await__` and drives what comes back - // by sending into it. both halves have edges worth pinning: what counts as - // awaitable at all, and what a finished delegation's value *is* - agree( - "awaitproto", +fn a_decorated_method_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); + return; + } + agree_with_declines( + "methoddeco", "\ -async def awaited(x: object) -> object: - return await x - -async def twice(x: object, y: object) -> object: - a = await x - b = await y - return (a, b) - -async def guarded(x: object) -> object: - try: - return await x - except ValueError as e: - return 'caught ' + str(e) - -def delegated(xs: object) -> object: - got = yield from xs - yield got +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper -def returns(v: object) -> object: - return v - yield +data class Point: + x: int + y: int -async def returns_async(v: object) -> object: - return v + @property + def total(self) -> int: + return self.x + self.y -def redelegated(v: object) -> object: - got = yield from returns(v) - yield got + @doubling + def raw(self) -> int: + return self.x ", &[ - // the *other* half of the same rule: what a compiled frame's own return - // value becomes on the way out. `StopIteration` reads its argument as an - // argument *list*, so a tuple would arrive spread and an exception - // instance would be raised in place of the `StopIteration` itself - "[_value(m.returns(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2], 'ab')]", - "[_run(m.returns_async(v)) for v in ((1, 2), (), (1,), 5, None, [1, 2])]", - "[list(m.redelegated(v)) for v in ((1, 2), (), (1,), 5, None)]", - "[_run(m.awaited(m.returns_async(v))) for v in ((1, 2), (), 5, None)]", - "_value(m.returns(ValueError('x')))", - "_value(m.returns(StopIteration(9)))", - "_run(m.returns_async(StopIteration(9)))", - // and the exception it rides on is shaped the way python shapes it - "[(lambda e: (repr(e), e.args))(_capture(next, m.returns(v))) \ - for v in ((1, 2), (), 5, None)]", - // an await that completes without ever suspending — the common case - "_run(m.awaited(_Ready(7)))", - "_run(m.twice(_Ready(1), _Ready(2)))", - // one that really suspends, driven by a loop that has to resume it - "_run(m.awaited(_sleeps(9)))", - "_run(m.awaited(_Suspends(11)))", - "_run(m.twice(_Ready(1), _sleeps(2)))", - "_run(m.twice(_Suspends(1), _Suspends(2)))", - // a bare `StopIteration` carries `None`, and a subclass carries its own - "_run(m.awaited(_Raises(StopIteration())))", - "_run(m.awaited(_Raises(StopIteration(3))))", - "_run(m.awaited(_Raises(_Sub(5))))", - // python reads the exception's field, not its `value` attribute - "_run(m.awaited(_Raises(_Shadowed(7))))", - "_Shadowed(7).value", - // a raised *type* rather than an instance still has to become one - "_run(m.awaited(_Raises(StopIteration)))", - // normalising a tuple argument would take only its first element - "_run(m.awaited(_Raises(StopIteration((1, 2)))))", - // anything else escaping the awaited object is not a return value - "_capture_async(m.awaited, _throws(ValueError))", - "_capture_async(m.awaited, _throws(KeyError))", - "_run(m.guarded(_throws(ValueError)))", - "_capture_async(m.guarded, _throws(KeyError))", - // and what is not awaitable at all says so, about the right object - "_capture_async(m.awaited, 5)", - "_capture_async(m.awaited, _counting(1))", - "_capture_async(m.awaited, _NotIter())", - "_capture_async(m.awaited, _CoroAwait())", - // `yield from` is the same machine over an iterator rather than an - // awaitable, and an exhausted one returns `None` rather than nothing - "list(m.delegated(iter([1, 2, 3])))", - "list(m.delegated(_counting(2)))", - "list(m.delegated(iter([])))", - "_capture(list, m.delegated(_RaiseIter(_Sub(5))))", - "_sent(m.delegated(_counting(2)), (7, 8))", + // a property is a descriptor on the type, reached without a call + "m.Point(3, 4).total", + "type(m.Point.total).__name__", + // and a user decorator wraps the native method + "m.Point(3, 4).raw()", ], ); } +/// a decorator that *mutates* what it is handed, next to one that wraps +/// +/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the +/// same object back, so it is the whole class of decorator a compiled method has to +/// stay writable for — a method descriptor takes no attributes at all. and the two +/// class constructions have to be covered separately: `Plain` is built from a spec, so +/// the decorators reach methods this module compiled, while `Shape`'s metaclass rules +/// a spec out and its construction falls back to the interpreted definition, which +/// already carries them #[test] -fn a_value_live_across_a_suspension_agrees() { - // python evaluates left to right, so `total + await step(i)` has the read of - // `total` on the stack while the `await` suspends. every shape here holds - // something across a suspension that has no name of its own, and `agree` is what - // says they compile rather than falling back - agree( - "parked", +fn a_mutating_method_decorator_agrees() { + agree_python( + "mutatingdeco", "\ -async def stepped(i: int) -> int: - return (i * 7) % 13 - -async def summed(n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + await stepped(i) - i = i + 1 - return total +from abc import ABC, abstractmethod -async def paired(n: int) -> int: - return (await stepped(n)) + (await stepped(n + 1)) -async def tripled(n: int) -> int: - return (await stepped(n)) * 100 + (await stepped(n + 1)) * 10 + (await stepped(n + 2)) +def doubling(fn: object) -> object: + def wrapper(self: object) -> object: + return fn(self) * 2 + return wrapper -def mixed(a: int, b: int, c: int) -> int: - return a * 100 + b * 10 + c -async def spanning(n: int) -> int: - return mixed(n * 5, await stepped(n), await stepped(n + 1)) +def tagging(fn: object) -> object: + def wrapper(self: object) -> object: + return str(fn(self)) + '!' + return wrapper -async def awaited(step: object, n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + await step(i) - i = i + 1 - return total -def echoing(n: int) -> object: - total = 0 - i = 0 - while i < n: - total = total + (yield i) - i = i + 1 - return total +class Plain: + @abstractmethod + def area(self) -> int: + return 3 -def twinned() -> object: - return (yield 1) + (yield 2) + @doubling + def raw(self) -> int: + return 5 -def straddling(n: int) -> object: - yield mixed(n * 5, (yield 1), (yield 2)) + @tagging + @doubling + def stacked(self) -> int: + return 4 -def recovering(n: int) -> object: - total = n * 2 - try: - yield total - except ValueError: - yield total + 1 - yield total + 2 + @property + def total(self) -> int: + return 6 -def crossed(xs: list[int], ys: list[int]) -> object: - for a in xs: - base = a * 100 - for b in ys: - yield base + (yield b) -def counting(n: int) -> object: - yield n - return n * 100 +class Shape(ABC): + @abstractmethod + def area(self) -> int: + return 7 -def relayed(n: int) -> object: - base = n + 1 - got = yield from counting(n) - yield base + got + @doubling + def raw(self) -> int: + return 11 ", &[ - "__import__('asyncio').run(m.summed(40))", - "__import__('asyncio').run(m.paired(3))", - "__import__('asyncio').run(m.tripled(3))", - "__import__('asyncio').run(m.spanning(3))", - // `n * 5` is read before the first suspension and used after the *second*, - // so nothing between the two reads it. the static flow calls it dead at the - // first — a suspension is a `return` and goes nowhere — and one field has to - // carry it across both - "_sent(m.straddling(3), (4, 5))", - // arbitrary precision survives the park, which a machine word would not - "__import__('asyncio').run(m.summed(3)) + 10 ** 30", - // awaiting something that *does* suspend, so the loop resumes the frame - "__import__('asyncio').run(m.awaited(_slow, 4))", - "_sent(m.echoing(4), (10, 20, 30, 40))", - "_sent(m.twinned(), (5, 7))", - "list(m.recovering(3))", - // a `throw` the handler catches, and the parked value read after it - "_recovered(m.recovering(3), ValueError, 2)", - "_sent(m.crossed([1, 2], [7, 8]), (100, 200, 300, 400, 500, 600, 700, 800))", - "list(m.relayed(4))", - // a coroutine driven by hand, with no loop underneath it - "[(c := m.summed(3), _capture(c.send, None).value)[1:]]", + // the mutating decorator ran, and the method it marked still calls + "m.Plain.area.__isabstractmethod__", + "m.Plain().area()", + // the name a decorator reads off a function is still the method's own + "m.Plain.area.__name__", + // a wrapping decorator wraps once, not once per construction + "m.Plain().raw()", + // and the innermost is applied first, so `tagging` sees the doubled value + "m.Plain().stacked()", + "m.Plain().total", + "type(m.Plain.total).__name__", + // an `ABCMeta` base reads the mark, so the abstract set is the same set + "sorted(m.Shape.__abstractmethods__)", + "m.Shape.area.__isabstractmethod__", + "type('S', (m.Shape,), {'area': lambda self: 13})().raw()", + "type('S', (m.Shape,), {'area': lambda self: 13})().area()", ], ); } #[test] -fn a_handled_throw_leaves_the_generator_usable() { - // the field carrying what `throw` wants raised starts null, and is *emptied* by - // writing `None` into it because no operation stores a null. reading it back as a - // second exception raised `SystemError` at every later resumption - agree( - "rethrown", +fn lambdas_agree() { + agree_with_declines( + "lambdas", "\ -def recovering(n: int) -> object: - try: - yield n - except ValueError: - pass - yield n + 2 - yield n + 3 +def adder(n: int) -> ((int) -> int): + return lambda x: x + n + +def twice(n: int) -> int: + f = lambda x: x * 2 + return f(f(n)) + +def picked(flag: bool) -> ((int) -> int): + if flag: + return lambda x: x + 1 + return lambda x: x - 1 + +def counter() -> (() -> int): + n = 0 + f = lambda: n + n = 1 + return f + +def each(values: list[int]) -> list[object]: + out = [] + for v in values: + out.append(lambda: v) + return out ", &[ - "_recovered(m.recovering(3), ValueError, 2)", - "list(m.recovering(3))", + "m.adder(3)(4)", + "m.twice(5)", + "(m.picked(True)(10), m.picked(False)(10))", + // the lambda closes over the *variable*, so it sees the later write + "m.counter()()", + // and every lambda a loop makes shares one cell, as in python + "[f() for f in m.each([1, 2, 3])]", + "sorted([3, 1, 2], key=m.adder(0))", + "[m.adder(a)(a) for a in (0, -3, 10 ** 20)]", ], ); } #[test] -fn a_parked_value_does_not_leak() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_parkleak"); - let _ = std::fs::remove_dir_all(&dir); - // `label + (yield i)` holds the label across the suspension, so the state object - // owns it for as long as the frame is parked - let source = "\ -def tagged(label: str, times: int) -> object: - i = 0 - while i < times: - yield label + (yield i) - i = i + 1 - -def handling(label: str) -> object: - try: - yield label - except ValueError: - yield label + \"!\" - yield label + \"?\" -"; - if build_source( - source, - "by_diff_parkleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); +fn a_loop_over_native_instances_agrees() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { + eprintln!("skipping: `data class` needs python 3.10"); return; } - // driven to exhaustion, and abandoned *while suspended* with the value parked - let out = run( - &python, - &dir, - "import gc, sys, by_diff_parkleak as m\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(20000):\n\ - \x20 g = m.tagged(label, 2)\n\ - \x20 next(g)\n\ - \x20 g.send('a')\n\ - \x20 g.close()\n\ - del g\n\ - gc.collect()\n\ - print('driven', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.tagged(label, 9)\n\ - \x20 next(g)\n\ - \x20 g.send('a')\n\ - del g\n\ - gc.collect()\n\ - print('parked', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(20000):\n\ - \x20 g = m.handling(label)\n\ - \x20 next(g)\n\ - \x20 g.throw(ValueError('boom'))\n\ - del g\n\ - gc.collect()\n\ - print('handled', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + // narrowing the element to an emitted class is a checked unbox like any other — + // asking the *free* narrowable rather than the one that knows this module's + // layouts declined the whole loop + agree( + "loopnative", + "\ +frozen data class Vec2: + x: float + y: float + +data class Loose: + n: int + +def total(vs: list[Vec2]) -> float: + out = 0.0 + for v in vs: + out = out + v.x * v.x + v.y * v.y + return out + +def summed(xs: list[Loose]) -> int: + return sum([x.n for x in xs]) + +def wrong_element(vs: list[Vec2]) -> float: + out = 0.0 + for v in vs: + out = out + v.x + return out +", + &[ + "round(m.total([m.Vec2(1.0, 2.0), m.Vec2(3.0, 4.0)]), 1)", + "m.summed([m.Loose(1), m.Loose(2)])", + "round(m.total([]), 1)", + // and the narrowing is a real check: a list that lies raises + "[type(e).__name__ for e in [_capture(m.wrong_element, ['not a vec'])]]", + ], ); - // a suspension inside `except` parks the exception the handler took over from, so - // the state object is holding one when it is dropped mid-handler - assert_eq!(out, "driven stable\nparked stable\nhandled stable"); } -#[test] -fn a_coroutine_is_awaitable_and_not_iterable() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_coro"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -async def plain(n: int) -> int: - return n * 2 +/// the process's own peak footprint, which is the only thing that sees a buffer +/// that is not a `PyObject` +/// +/// windows has no `resource` module; `GetProcessMemoryInfo` answers the same +/// question there. the structure is spelled by field *width* rather than by C +/// name because `ctypes.c_ulong` is four bytes on windows and eight elsewhere — +/// a `DWORD` written as `c_ulong` would move every offset behind it — and its +/// size is checked on every platform, so the branch only windows takes is still +/// held to its layout here +const PEAK_FOOTPRINT: &str = "\ +import ctypes + +class _Counters(ctypes.Structure): + _fields_ = [(\"cb\", ctypes.c_uint32), (\"page_faults\", ctypes.c_uint32)] + [ + (name, ctypes.c_size_t) + for name in (\"peak_working_set\", \"working_set\", + \"quota_peak_paged_pool\", \"quota_paged_pool\", + \"quota_peak_non_paged_pool\", \"quota_non_paged_pool\", + \"pagefile\", \"peak_pagefile\")] + +assert ctypes.sizeof(_Counters) == 8 + 8 * ctypes.sizeof(ctypes.c_size_t) + +def _windows_peak(): + kernel32 = ctypes.windll.kernel32 + kernel32.GetCurrentProcess.restype = ctypes.c_void_p + kernel32.K32GetProcessMemoryInfo.restype = ctypes.c_int32 + kernel32.K32GetProcessMemoryInfo.argtypes = ( + ctypes.c_void_p, ctypes.POINTER(_Counters), ctypes.c_uint32) + counters = _Counters() + counters.cb = ctypes.sizeof(counters) + if not kernel32.K32GetProcessMemoryInfo( + kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): + raise ctypes.WinError() + return counters.peak_working_set + +try: + import resource +except ImportError: + _peak = _windows_peak +else: + def _peak(): + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss "; - if build_source( - source, - "by_diff_coro", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // `asyncio.iscoroutine` tests the abc, and iterating a coroutine has to be an - // error rather than quietly working - let out = run( - &python, - &dir, - "import asyncio, collections.abc as abc, by_diff_coro as m\n\ - c = m.plain(2)\n\ - print(asyncio.iscoroutine(c), isinstance(c, abc.Coroutine))\n\ - print(hasattr(c, 'send'), hasattr(c, 'throw'), hasattr(c, 'close'))\n\ - try:\n list(c)\n\ - except TypeError:\n print('not iterable')\n\ - print(asyncio.run(c))\n", - ); - assert_eq!(out, "True True\nTrue True True\nnot iterable\n4"); -} #[test] -fn a_coroutine_does_not_leak() { +fn an_unboxed_array_does_not_leak_its_buffer() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_coroleak"); + let dir = std::env::temp_dir().join("by_diff_arrayleak"); let _ = std::fs::remove_dir_all(&dir); + // the buffer is `PyMem_Malloc`, not a `PyObject` — so a leak of one is invisible + // to `gc.get_objects()` and to a refcount check. the process's own footprint is + // the only thing that sees it let source = "\ -async def echo(label: str) -> str: - return label +def churn(rounds: int) -> float: + out = 0.0 + r = 0 + while r < rounds: + xs = [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] + out = out + xs[0] + r = r + 1 + return out "; if build_source( source, - "by_diff_coroleak", + "by_diff_arrayleak", &toolchain, &dir, &Options::default(), @@ -4683,1843 +6816,1918 @@ async def echo(label: str) -> str: eprintln!("skipping: no working C toolchain"); return; } - // including a coroutine that is created and never awaited let out = run( &python, &dir, - "import asyncio, gc, sys, warnings, by_diff_coroleak as m\n\ - warnings.simplefilter('ignore')\n\ - label = 'x' * 40\n\ - before = sys.getrefcount(label)\n\ - for _ in range(5000):\n\ - \x20 asyncio.run(m.echo(label))\n\ - print('awaited', 'stable' if sys.getrefcount(label) == before else 'leaked')\n\ - for _ in range(5000):\n\ - \x20 c = m.echo(label)\n\ - del c\n\ - gc.collect()\n\ - print('abandoned', 'stable' if sys.getrefcount(label) == before else 'leaked')\n", + &format!( + "import by_diff_arrayleak as m\n{PEAK_FOOTPRINT}\ + m.churn(1000)\n\ + before = _peak()\n\ + m.churn(400000)\n\ + after = _peak()\n\ + print(after - before < before // 2 or after - before < 4_000_000)\n" + ), ); - assert_eq!(out, "awaited stable\nabandoned stable"); + assert_eq!(out, "True", "the array buffer leaks"); } #[test] -fn with_blocks_agree() { - agree_with_declines( - "withblocks", +fn container_fast_paths_agree_and_respect_subclasses() { + // the fast paths are guarded on the *exact* type: a subclass may override + // `__getitem__` or `__missing__`, and a fast path that ignored that would be a + // wrong answer rather than a fast one + agree( + "containerprim", "\ -def counted(path: str) -> int: - total = 0 - with open(path) as f: - for line in f: - total = total + 1 - return total - -def guarded(mgr: object) -> str: - with mgr: - return \"body\" - return \"after\" +def indexed(xs: list[int], i: int) -> int: + return xs[i] -def nested(mgr: object, other: object) -> str: - with mgr: - with other: - return \"inner\" - return \"after\" +def written(xs: list[int], i: int, v: int) -> str: + xs[i] = v + return str(xs) -def suppressing(mgr: object) -> str: - with mgr: - raise ValueError(\"boom\") - return \"suppressed\" +def looked_up(d: dict[str, int], k: str) -> int: + return d[k] -def propagating(mgr: object) -> str: - with mgr: - raise ValueError(\"boom\") - return \"not reached\" +def sized(xs: object) -> int: + return len(xs) -def entered(mgr: object) -> object: - with mgr as value: - return value - return None +def tupled(t: tuple[int, int], i: int) -> object: + return t[i] ", &[ - // `__exit__` runs on the normal path with `(None, None, None)`, and a - // `return` from inside the body is one of those paths - "_run_recording(m)", - "m.suppressing(_Swallow())", - "[type(e).__name__ for e in [_capture(m.propagating, _Pass())]]", - "m.entered(_Value(7))", - "_run_nested(m)", + "m.indexed([1, 2, 3], 1)", + "m.indexed([1, 2, 3], -1)", + "m.written([1, 2, 3], 0, 9)", + "m.looked_up({'a': 1}, 'a')", + "m.sized([1, 2, 3])", + "m.sized({'a': 1})", + "m.sized('abc')", + "m.tupled((4, 5), 1)", + // every error is python's own, message and class included + "[(type(e).__name__, str(e)) for e in [_capture(m.indexed, [1], 5)]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.written, [1], 5, 0)]]", + "[(type(e).__name__, repr(str(e))) for e in [_capture(m.looked_up, {}, 'zz')]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.tupled, (1,), 9)]]", + // a *subclass* overriding the protocol must still be honoured + "m.indexed(type('L', (list,), {'__getitem__': lambda s, i: 99})([1, 2]), 0)", + "m.looked_up(type('D', (dict,), {'__missing__': lambda s, k: 77})(), 'nope')", + "m.sized(type('L', (list,), {'__len__': lambda s: 42})([1]))", ], ); } #[test] -fn an_early_exit_runs_the_finally_it_is_leaving() { - // this was a silent wrong answer in a shipped feature: a `return` or a `break` - // inside `try` skipped the `finally` entirely - agree( - "unwind", +fn a_comprehension_over_range_agrees() { + // the statement form has had a counting loop all along; the comprehension drove + // the iteration protocol instead — a `range` object, an iterator, a `next` and + // an unbox *per element*, for a loop whose bounds are right there + agree_with_declines( + "comprange", "\ -def early(log: list[str]) -> str: - try: - return \"body\" - finally: - log.append(\"finally\") +def squares(n: int) -> object: + return [i * i for i in range(n)] -def looped(log: list[str], n: int) -> str: - i = 0 - while i < n: - try: - if i == 1: - break - finally: - log.append(\"f\") - i = i + 1 - return \"done\" +def filtered(n: int) -> object: + return [i for i in range(n) if i % 3 == 0] -def continued(log: list[str], n: int) -> int: - total = 0 - i = 0 - while i < n: - i = i + 1 - try: - if i == 2: - continue - total = total + i - finally: - log.append(\"f\") - return total +def from_two(n: int) -> object: + return [i for i in range(2, n)] -def layered(log: list[str]) -> str: - try: - try: - return \"deep\" - finally: - log.append(\"inner\") - finally: - log.append(\"outer\") +def nested(n: int) -> object: + return [i * j for i in range(n) for j in range(n)] + +def buffered(n: int) -> float: + xs = [i * 1.5 for i in range(n)] + out = 0.0 + for x in xs: + out = out + x + return out + +def empty(n: int) -> float: + xs = [i * 1.5 for i in range(0)] + out = 0.0 + for x in xs: + out = out + x + return out ", &[ - "[(log := [], m.early(log), log)[1:]]", - "[(log := [], m.looped(log, 3), log)[1:]]", - "[(log := [], m.continued(log, 4), log)[1:]]", - // innermost first, and both of them - "[(log := [], m.layered(log), log)[1:]]", + "m.squares(5)", + "m.filtered(10)", + "m.from_two(6)", + "m.nested(3)", + "m.squares(0)", + "m.from_two(1)", + "m.buffered(5)", + "m.empty(0)", + // a negative bound is an empty range, not a backwards one + "m.squares(-3)", ], ); } #[test] -fn parameter_defaults_agree() { +fn a_loop_over_an_unboxed_array_agrees() { + // the shape the whole representation exists for: an `i64` counter, no iterator + // object, no null test per step, and no bounds check — the counter is the + // lowering's own, so it is in range by construction agree_with_declines( - "defaults", + "arrayloop", "\ -def greet(name: str, greeting: str = \"hi\", times: int = 1) -> str: - return (greeting + \" \" + name) * times - -def offset(a: int, b: int = 10) -> int: - return a + b - -def flagged(a: int, on: bool = True) -> int: - if on: - return a - return -a - -def boxed_none(a: object = None) -> object: - return a +def total(n: int) -> float: + xs = [1.5, 2.5, 3.5, 4.5] + out = 0.0 + for x in xs: + out = out + x + return out -def boxed_int(a: object = 7) -> object: - return a +def empty(n: int) -> float: + xs = [1.0] + out = 0.0 + for x in xs: + out = out + x + return out -def boxed_bool(a: object = True) -> object: - return a +def broken(n: int) -> float: + xs = [1.0, 2.0, 3.0] + out = 0.0 + for x in xs: + if x > 1.5: + break + out = out + x + else: + out = -1.0 + return out -def boxed_float(a: object = 1.5) -> object: - return a +def skipped(n: int) -> float: + xs = [1.0, 2.0, 3.0] + out = 0.0 + for x in xs: + if x > 1.5: + continue + out = out + x + return out -def optional(a: int, extra: object = None) -> object: - if extra is None: - return a - return extra +def exhausted(n: int) -> float: + xs = [1.0, 2.0] + out = 0.0 + for x in xs: + out = out + x + else: + out = out + 100.0 + return out -def computed(a: int, b: object = []) -> object: - return b +def flags(n: int) -> int: + bs = [True, False, True, True] + total = 0 + for b in bs: + if b: + total = total + 1 + return total ", &[ - "m.greet('a')", - "m.greet('a', 'yo')", - "m.greet('a', 'yo', 2)", - "m.offset(1)", - "m.offset(1, 2)", - "(m.flagged(3), m.flagged(3, False))", - "(m.optional(1), m.optional(1, 'x'))", - // too few arguments still raises, and so does too many - "[type(e).__name__ for e in [_capture(m.offset)]]", - "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", - // a computed default declines, and the interpreted twin keeps the identity - "m.computed(1)", - // an immediate written into an *object* place has to be boxed: the - // unboxed `None` is a bare byte, and `By_NewRef` of one is a NULL the - // error check reads as a failure with no exception behind it - "m.boxed_none()", - "m.boxed_none(1)", - "m.boxed_int()", - "m.boxed_bool()", - "m.boxed_float()", - "(m.boxed_none() is None, m.boxed_bool() is True)", + "m.total(0)", + "m.empty(0)", + // `break` skips the `else`, `continue` does not + "m.broken(0)", + "m.skipped(0)", + "m.exhausted(0)", + "m.flags(0)", ], ); } #[test] -fn a_default_filled_at_a_native_call_site_agrees() { - // `parameter_defaults_agree` calls these from python, which fills a default in the - // *wrapper* — where the value is already an object. a compiled caller fills it - // inline instead, and that path pushed the default with no coercion at all: a bare - // `length=0` reaching an unannotated parameter put a tagged integer where the - // callee declares a `PyObject *`. it was `shutil.py` failing to build that found it, - // and only because the two representations differ enough for a C compiler to object +fn a_list_that_escapes_keeps_being_a_list() { + // the buffer is an optimization, not a restriction: a name that leaves the + // function never earns one in the first place, so it compiles exactly as it did + // before the representation existed agree( - "native_defaults", + "bufferescape", "\ -def taking(a, b, length=0, name=\"n\", scale=1.5, on=True, extra=None): - return (a, b, length, name, scale, on, extra) - - -def calling(x: int) -> object: - return taking(x, x) - - -def partly(x: int) -> object: - return taking(x, x, 3) - - -def by_keyword(x: int) -> object: - return taking(x, x, on=False) +def returned(n: int) -> object: + xs = [1.0, 2.0] + return xs +def passed(n: int) -> int: + xs = [1.0, 2.0] + return len(sorted(xs)) -def annotated(a: int, step: int = 2) -> int: - return a + step +def stored(n: int) -> object: + xs = [1.0, 2.0] + return [xs, xs] +def kept(n: int) -> float: + xs = [1.0, 2.0] + return xs[0] + xs[1] -def calling_annotated(x: int) -> int: - return annotated(x) +def looped(n: int) -> float: + xs = [1.0, 2.0] + out = 0.0 + for x in xs: + out = out + x + return out ", &[ - "m.calling(1)", - "m.partly(2)", - "m.by_keyword(3)", - "[m.calling_annotated(n) for n in (0, 5)]", - "m.taking(1, 2)", - "(m.calling(1)[3], type(m.calling(1)[4]).__name__, m.calling(1)[6] is None)", + // these keep a real list, and a real list is what comes back + "m.returned(0)", + "type(m.returned(0)).__name__", + "m.passed(0)", + "m.stored(0)", + // and these earn the buffer, invisibly + "m.kept(0)", + "m.looped(0)", ], ); } #[test] -fn a_string_default_does_not_leak() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_defaultleak"); - let _ = std::fs::remove_dir_all(&dir); - let source = "\ -def padded(a: str, fill: str = \"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\") -> str: - return a + fill -"; - if build_source( - source, - "by_diff_defaultleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the wrapper releases its arguments, so a default handed over borrowed would be - // released twice — and one handed over with an extra reference would leak - let out = run( - &python, - &dir, - "import sys, by_diff_defaultleak as m\n\ - fill = 'x' * 40\n\ - before = sys.getrefcount(fill)\n\ - for _ in range(20000):\n\ - \x20 m.padded('a')\n\ - print('stable' if sys.getrefcount(fill) == before else 'moved')\n\ - print(m.padded('a')[:3])\n", - ); - assert_eq!(out, "stable\naxx"); -} - -#[test] -fn keyword_arguments_agree() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree_with_declines( - "keywords", +fn a_from_import_inside_a_body_agrees() { + // plain python, so the interpreted leg is the source itself: the transpiler's + // lazy-import polyfill resolves `from pkg import submodule` wrongly, which would + // otherwise break the leg this is measured against rather than the compiled one + agree_python( + "fromimport", "\ -def offset(a: int, b: int = 10) -> int: - return a + b +def one(n: int) -> str: + from math import sqrt + return str(round(sqrt(n), 3)) + +def several(s: str) -> str: + from os.path import basename, dirname + return dirname(s) + '|' + basename(s) + +def aliased(n: int) -> str: + from math import sqrt as root + return str(round(root(n), 3)) -def described(name: str, sep: str = \"-\", times: int = 1) -> str: - return (name + sep) * times +# `urllib.parse` is not an attribute of `urllib` until something imports it, so +# this is the fromlist doing its job rather than an attribute read +def submodule(s: str) -> str: + from urllib import parse + return parse.quote(s) -def caller(a: int) -> int: - return offset(a, b=100) + offset(b=1, a=a) + offset(a) +def dotted(s: str) -> str: + from urllib.parse import quote + return quote(s) -data class Point: - x: int - y: int +# a name the module does not have is an `ImportError`, not the `AttributeError` a +# plain attribute read would give — a guarded lazy import rests on the difference +def absent() -> object: + from os import definitely_not_a_real_name + return definitely_not_a_real_name - def shifted(self, dx: int = 0, dy: int = 0) -> int: - return self.x + dx + self.y + dy +def guarded() -> str: + try: + from os import definitely_not_a_real_name + except ImportError: + return 'caught' + return 'no' ", &[ - // a python caller, by name and by position - "m.offset(1, b=2)", - "m.offset(b=2, a=1)", - "m.offset(1)", - "m.described('a', times=3)", - "m.described(name='a', sep='+', times=2)", - // a compiled caller resolves the names against the callee's signature - "m.caller(1)", - // and a method's keywords work through the type object - "m.Point(1, 2).shifted(dy=10)", - "m.Point(1, 2).shifted(3, dy=10)", - // the error cases match - "[type(e).__name__ for e in [_capture(m.offset, 1, 2, 3)]]", - "[str(_capture_kw(m.offset, (1,), {'a': 2}))]", - "[str(_capture_kw(m.offset, (), {'zzz': 1}))]", - "[str(_capture(m.offset))]", + "m.one(2)", + "m.several('/a/b/c.txt')", + "m.aliased(2)", + "m.submodule('a b/c')", + "m.dotted('a b/c')", + "m.guarded()", + // the type, the message and every attribute an `except` clause reads + "(lambda e: (type(e).__name__, str(e), e.name, e.path, e.name_from))(_capture(m.absent))", + // a thousand imports must not accumulate references to the module. a + // leaked module is one object with a climbing refcount, which no + // object-count check can see + "_repeated(lambda: m.one(2), 1000)", + "_refdelta('math', lambda: m.one(2), 500)", + "_refdelta('urllib.parse', lambda: m.submodule('a b'), 500)", ], ); } #[test] -fn variadic_parameters_agree() { - agree_with_declines( - "variadic", +fn the_remaining_statement_and_expression_forms_agree() { + agree( + "coverage", "\ -def total(*values: int) -> int: - out = 0 - for v in values: - out = out + v - return out +def imports(n: int) -> str: + import math + return str(round(math.sqrt(n), 3)) -def named(prefix: str, *rest: str) -> str: - out = prefix - for r in rest: - out = out + r - return out +def aliased(n: int) -> str: + import os.path as p + return str(p.basename('/a/b')) -def options(a: int, **rest: object) -> int: - return a + len(rest) +def del_key(d: dict[str, int]) -> str: + del d['a'] + return str(sorted(d.items())) -def both(a: int, *rest: int, **named: object) -> int: - return a + len(rest) + len(named) +def del_item(xs: list[int]) -> str: + del xs[0] + return str(xs) -def tupled(*values: int) -> object: - return values +def del_attr(o: object) -> str: + del o.gone + return str(hasattr(o, 'gone')) -def mapped(**named: object) -> object: - return named +def ellipsis(n: int) -> object: + return ... -def calls_total(a: int, b: int) -> int: - return total(a, b, 3) +def negated(s: object) -> object: + return -s -def calls_options(a: int) -> int: - return options(a, x=1, y=2) +def inverted(s: object) -> object: + return ~s -def calls_both(a: int) -> int: - return both(a, 1, 2, k=3, j=4) +def sliced(xs: list[int]) -> str: + return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[:2]) + str(xs[::-1]) -def calls_none() -> int: - return total() +def slice_assigned(xs: list[int]) -> str: + xs[1:3] = [9, 9, 9] + return str(xs) + +def slice_deleted(xs: list[int]) -> str: + del xs[1:3] + return str(xs) + +def walrus(xs: list[int]) -> str: + if (n := len(xs)) > 2: + return 'big ' + str(n) + return 'small ' + str(n) + +def genexp(xs: list[int]) -> str: + return str(sum(x * 2 for x in xs)) + str(any(x > 2 for x in xs)) + str(max(x for x in xs)) + +def declared_global(n: int) -> int: + global _counter + return n ", &[ - "m.total()", - "m.total(1, 2, 3)", - "(m.named('a'), m.named('a', 'b', 'c'))", - "(m.options(1), m.options(1, x=2, y=3))", - "m.both(1, 2, 3, k=4)", - // the body sees a real tuple and a real dict - "m.tupled(1, 2)", - "sorted(m.mapped(b=2, a=1).items())", - "type(m.tupled()).__name__", - "type(m.mapped()).__name__", - // a keyword that names no parameter goes to `**kwargs`, or raises - "[str(_capture_kw(m.named, ('a',), {'zzz': 1}))]", - "[str(_capture(m.options))]", - "[m.total(*a) for a in ([1], [1, 2])]", - // and a *compiled* caller packs the tuple and the dict itself - "m.calls_total(1, 2)", - "m.calls_options(5)", - "m.calls_both(1)", - "m.calls_none()", + "m.imports(16)", + "m.aliased(0)", + "m.del_key({'a': 1, 'b': 2})", + "m.del_item([1, 2, 3])", + "m.ellipsis(0)", + "m.sliced([1, 2, 3, 4])", + "m.slice_assigned([1, 2, 3, 4])", + "m.slice_deleted([1, 2, 3, 4])", + "m.walrus([1, 2, 3])", + "m.walrus([1])", + "m.genexp([1, 2, 3])", + "m.declared_global(4)", + // the protocol forms, on a type that answers them + "m.negated(type('N', (), {'__neg__': lambda s: 'neg'})())", + "m.inverted(type('N', (), {'__invert__': lambda s: 'inv'})())", + "m.del_attr(type('A', (), {})() if False else __import__('types').SimpleNamespace(gone=1))", + // and the errors each raises + "[(type(e).__name__, str(e)) for e in [_capture(m.del_key, {})]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.del_item, [])]]", + "[(type(e).__name__, str(e)) for e in [_capture(m.negated, object())]]", ], ); } #[test] -fn a_variadic_argument_does_not_leak() { +fn a_bad_first_argument_raises_rather_than_crashing() { + // the wrapper releases every argument local on the error path, so one whose + // declaration a `goto` skipped would be released while indeterminate. a wrong + // type in the *first* parameter is the reachable case: it jumps over the rest. + // + // only the exception *type* is compared: the boundary rejects a bad argument + // where the interpreted leg gets as far as the operation that uses it, so the + // two agree that it is a `TypeError` and not on where it was raised + agree( + "badarg", + "\ +def two(a: int, b: str) -> int: + return a + len(b) + +def three(a: int, b: str, c: list[int]) -> int: + return a + len(b) + len(c) +", + &[ + "type(_capture(m.two, 'x', 'y')).__name__", + "type(_capture(m.two, 1, 2)).__name__", + "type(_capture(m.three, 'x', 'y', [1])).__name__", + "type(_capture(m.three, 1, 'y', 'z')).__name__", + "m.two(1, 'yy')", + "m.three(1, 'yy', [1, 2])", + ], + ); +} + +#[test] +fn a_plain_python_loop_closure_shares_its_binding() { let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_varleak"); - let _ = std::fs::remove_dir_all(&dir); + // python's loop binding is shared by every iteration, so all three closures + // see the last value. basedpython's is per-iteration. the compiled half has to + // follow the *source* language, not a flag — a `.py` fallback is python let source = "\ -def joined(*parts: str) -> int: - total = 0 - for p in parts: - total = total + len(p) - return total - -def mapped(**named: object) -> int: - return len(named) -"; - if build_source( - source, - "by_diff_varleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - // the wrapper *builds* the tuple and the dict, so it owns them and has to release - // them — and the elements it put in are borrowed from the caller - let out = run( - &python, - &dir, - "import gc, sys, by_diff_varleak as m\n\ - part = 'x' * 40\n\ - before = sys.getrefcount(part)\n\ - for _ in range(20000):\n\ - \x20 m.joined(part, part)\n\ - \x20 m.mapped(a=part)\n\ - print('refs', 'stable' if sys.getrefcount(part) == before else 'leaked')\n\ - gc.collect()\n\ - objects = len(gc.get_objects())\n\ - for _ in range(5000):\n\ - \x20 m.joined(part)\n\ - \x20 m.mapped(a=part, b=part)\n\ - gc.collect()\n\ - print('objects', 'stable' if len(gc.get_objects()) <= objects else 'leaked')\n", - ); - assert_eq!(out, "refs stable\nobjects stable"); +def counters() -> list[object]: + out = [] + for i in range(3): + def get() -> int: + return i + out.append(get) + return [f() for f in out] +"; + let dir = std::env::temp_dir().join("by_diff_pyloop"); + let interpreted = std::env::temp_dir().join("by_diff_pyloop_i"); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&interpreted); + std::fs::create_dir_all(&interpreted).expect("the directory is created"); + std::fs::write(interpreted.join("by_diff_pyloop.py"), source).expect("written"); + + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + let built = match build_source(source, "by_diff_pyloop", &toolchain, &dir, &options) { + Ok(built) => built, + Err(error) => { + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); + + let body = "import by_diff_pyloop as m\nprint(m.counters())\n"; + let compiled = run(&python, &dir, body); + assert_eq!(compiled, run(&python, &interpreted, body)); + assert!(compiled.contains("[2, 2, 2]"), "{compiled}"); } #[test] -fn a_decorated_method_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - agree_with_declines( - "methoddeco", +fn dunder_methods_fill_their_type_slots() { + // a method table cannot fill a slot: `repr(x)` reads `tp_repr` and never looks + // the name up. so each of these has to work *both* ways — through the slot and + // as an ordinary method + agree_python( + "dunders", "\ -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +class Bag: + def __init__(self, items: list[int]) -> None: + self.items = items -data class Point: - x: int - y: int + def __repr__(self) -> str: + return 'Bag(' + str(self.items) + ')' - @property - def total(self) -> int: - return self.x + self.y + def __str__(self) -> str: + return 'bag of ' + str(len(self.items)) - @doubling - def raw(self) -> int: - return self.x + def __len__(self) -> int: + return len(self.items) + + def __bool__(self) -> bool: + return len(self.items) > 1 ", &[ - // a property is a descriptor on the type, reached without a call - "m.Point(3, 4).total", - "type(m.Point.total).__name__", - // and a user decorator wraps the native method - "m.Point(3, 4).raw()", + "repr(m.Bag([1, 2, 3]))", + "str(m.Bag([1, 2, 3]))", + "len(m.Bag([1, 2, 3]))", + "bool(m.Bag([1, 2, 3]))", + "bool(m.Bag([1]))", + "len(m.Bag([]))", + "bool(m.Bag([]))", + "f'{m.Bag([1])}'", + "'{}'.format(m.Bag([1, 2]))", + "m.Bag([1, 2]).__repr__()", + "m.Bag([1, 2]).__len__()", + "[b for b in [m.Bag([]), m.Bag([1, 2])] if b]", ], ); } -/// a decorator that *mutates* what it is handed, next to one that wraps -/// -/// `abc.abstractmethod` writes `__isabstractmethod__` onto its argument and hands the -/// same object back, so it is the whole class of decorator a compiled method has to -/// stay writable for — a method descriptor takes no attributes at all. and the two -/// class constructions have to be covered separately: `Plain` is built from a spec, so -/// the decorators reach methods this module compiled, while `Shape`'s metaclass rules -/// a spec out and its construction falls back to the interpreted definition, which -/// already carries them #[test] -fn a_mutating_method_decorator_agrees() { +fn calling_a_resumable_frame_hands_back_the_state_object() { + // whatever the annotation says the body produces, the *call* gives the state + // object and the iteration or the `await` turns it back. there are three + // signature maps — module-level, methods, nested — and this has to hold in all of + // them, or the caller assigns a `PyObject *` into the annotation's representation agree_python( - "mutatingdeco", + "resumablecall", "\ -from abc import ABC, abstractmethod - +from typing import Any -def doubling(fn: object) -> object: - def wrapper(self: object) -> object: - return fn(self) * 2 - return wrapper +def flags(n): + i = 0 + while i < n: + yield i > 0 + i = i + 1 -def tagging(fn: object) -> object: - def wrapper(self: object) -> object: - return str(fn(self)) + '!' - return wrapper +async def doubled(n: int) -> int: + return n * 2 -class Plain: - @abstractmethod - def area(self) -> int: - return 3 - @doubling - def raw(self) -> int: - return 5 +class Holder: + def __init__(self, n: int) -> None: + self.n = n - @tagging - @doubling - def stacked(self) -> int: - return 4 + def counted(self): + i = 0 + while i < self.n: + yield i == 0 + i = i + 1 - @property def total(self) -> int: - return 6 + out = 0 + for seen in self.counted(): + if seen: + out = out + 1 + return out -class Shape(ABC): - @abstractmethod - def area(self) -> int: - return 7 +def nested(n: int) -> int: + def inner(): + i = 0 + while i < n: + yield i < 1 + i = i + 1 + out = 0 + for seen in inner(): + if seen: + out = out + 1 + return out - @doubling - def raw(self) -> int: - return 11 + +def drained(n: int) -> int: + out = 0 + for seen in flags(n): + if seen: + out = out + 1 + return out ", &[ - // the mutating decorator ran, and the method it marked still calls - "m.Plain.area.__isabstractmethod__", - "m.Plain().area()", - // the name a decorator reads off a function is still the method's own - "m.Plain.area.__name__", - // a wrapping decorator wraps once, not once per construction - "m.Plain().raw()", - // and the innermost is applied first, so `tagging` sees the doubled value - "m.Plain().stacked()", - "m.Plain().total", - "type(m.Plain.total).__name__", - // an `ABCMeta` base reads the mark, so the abstract set is the same set - "sorted(m.Shape.__abstractmethods__)", - "m.Shape.area.__isabstractmethod__", - "type('S', (m.Shape,), {'area': lambda self: 13})().raw()", - "type('S', (m.Shape,), {'area': lambda self: 13})().area()", + "list(m.flags(3))", + "m.drained(3)", + "m.drained(0)", + "_run(m.doubled(5))", + "m.Holder(4).total()", + "list(m.Holder(2).counted())", + "m.nested(3)", + "m.nested(0)", ], ); } #[test] -fn lambdas_agree() { - agree_with_declines( - "lambdas", +fn a_class_of_only_methods_compiles() { + // no `__init__` is not the same as no *layout*: a class of methods has an empty + // one, which is as representable as any other. `object.__init__` is what rejects a + // call with arguments, and it names the class rather than a method it does not have + // + // `Empty` is where the class is a *static* type, which nothing can build on, so it + // fills the pair of slots itself and raises through them. `Consts` is a base, so it + // is a heap type and leaves both to `object` — where the message carries the module, + // because a type built from a spec keeps its module in `tp_name`. that difference is + // the price of not publishing an `__init__` the source never wrote: one installed + // anyway stands between everything built on the class and the `object.__init__` it + // should have reached + agree_python( + "onlymethods", "\ -def adder(n: int) -> ((int) -> int): - return lambda x: x + n +class Helpers: + def doubled(self, n: int) -> int: + return n * 2 -def twice(n: int) -> int: - f = lambda x: x * 2 - return f(f(n)) + def summed(self, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + i + i = i + 1 + return total -def picked(flag: bool) -> ((int) -> int): - if flag: - return lambda x: x + 1 - return lambda x: x - 1 -def counter() -> (() -> int): - n = 0 - f = lambda: n - n = 1 - return f +class Consts: + SCALE = 3 -def each(values: list[int]) -> list[object]: - out = [] - for v in values: - out.append(lambda: v) - return out -", - &[ - "m.adder(3)(4)", - "m.twice(5)", - "(m.picked(True)(10), m.picked(False)(10))", - // the lambda closes over the *variable*, so it sees the later write - "m.counter()()", - // and every lambda a loop makes shares one cell, as in python - "[f() for f in m.each([1, 2, 3])]", - "sorted([3, 1, 2], key=m.adder(0))", - "[m.adder(a)(a) for a in (0, -3, 10 ** 20)]", + def scaled(self, n: int) -> int: + return n * Consts.SCALE + + +class Empty: + pass + + +class Sub(Consts): + def twice(self, n: int) -> int: + return self.scaled(n) * 2 +", + &[ + "m.Helpers().doubled(21)", + "m.Helpers().summed(5)", + "m.Consts().scaled(4)", + "m.Consts.SCALE", + "m.Sub().twice(4)", + "isinstance(m.Sub(), m.Consts)", + "type(m.Empty()).__name__", + "type(_capture(lambda: m.Consts(1))).__name__", + "str(_capture(lambda: m.Consts(1))).rsplit('.', 1)[-1]", + "str(_capture(lambda: m.Empty(1, 2)))", + // and a class anything can be built on publishes no `__init__` the source + // did not write. a static type still does, because it has to fill the slot + // pair itself — but nothing can be built on one, so no mro reaches it + "'__init__' in vars(m.Consts)", + "[m.Helpers().doubled(n) for n in [0, 1, 2]]", ], ); } #[test] -fn a_loop_over_native_instances_agrees() { - if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 10))) { - eprintln!("skipping: `data class` needs python 3.10"); - return; - } - // narrowing the element to an emitted class is a checked unbox like any other — - // asking the *free* narrowable rather than the one that knows this module's - // layouts declined the whole loop - agree( - "loopnative", +fn an_explicit_object_base_is_no_base_at_all() { + // `class C(object)` is what `class C:` already is, so it lays out and compiles the + // same way — but only when `object` really is the builtin + agree_python( + "objectbase", "\ -frozen data class Vec2: - x: float - y: float +class Plain(object): + def __init__(self, n: int) -> None: + self.n = n -data class Loose: - n: int + def doubled(self) -> int: + return self.n * 2 -def total(vs: list[Vec2]) -> float: - out = 0.0 - for v in vs: - out = out + v.x * v.x + v.y * v.y - return out -def summed(xs: list[Loose]) -> int: - return sum([x.n for x in xs]) +class Derived(Plain): + def __init__(self, n: int) -> None: + self.n = n + self.extra = 1 -def wrong_element(vs: list[Vec2]) -> float: - out = 0.0 - for v in vs: - out = out + v.x - return out + def total(self) -> int: + return self.doubled() + self.extra ", &[ - "round(m.total([m.Vec2(1.0, 2.0), m.Vec2(3.0, 4.0)]), 1)", - "m.summed([m.Loose(1), m.Loose(2)])", - "round(m.total([]), 1)", - // and the narrowing is a real check: a list that lies raises - "[type(e).__name__ for e in [_capture(m.wrong_element, ['not a vec'])]]", + "m.Plain(3).doubled()", + "m.Derived(4).total()", + "isinstance(m.Derived(1), m.Plain)", + "[m.Plain(n).doubled() for n in [0, 1, 2]]", + "m.Derived(2).n", ], ); } -/// the process's own peak footprint, which is the only thing that sees a buffer -/// that is not a `PyObject` -/// -/// windows has no `resource` module; `GetProcessMemoryInfo` answers the same -/// question there. the structure is spelled by field *width* rather than by C -/// name because `ctypes.c_ulong` is four bytes on windows and eight elsewhere — -/// a `DWORD` written as `c_ulong` would move every offset behind it — and its -/// size is checked on every platform, so the branch only windows takes is still -/// held to its layout here -const PEAK_FOOTPRINT: &str = "\ -import ctypes - -class _Counters(ctypes.Structure): - _fields_ = [(\"cb\", ctypes.c_uint32), (\"page_faults\", ctypes.c_uint32)] + [ - (name, ctypes.c_size_t) - for name in (\"peak_working_set\", \"working_set\", - \"quota_peak_paged_pool\", \"quota_paged_pool\", - \"quota_peak_non_paged_pool\", \"quota_non_paged_pool\", - \"pagefile\", \"peak_pagefile\")] +#[test] +fn a_module_that_binds_object_itself_keeps_its_own() { + // the name is resolved, not matched: a class of this module's own called `object` + // is the base, and taking the builtin instead would give the subclass the wrong one + agree_python( + "shadowedobject", + "\ +class object: + def __init__(self, n: int) -> None: + self.n = n -assert ctypes.sizeof(_Counters) == 8 + 8 * ctypes.sizeof(ctypes.c_size_t) + def base(self) -> int: + return self.n -def _windows_peak(): - kernel32 = ctypes.windll.kernel32 - kernel32.GetCurrentProcess.restype = ctypes.c_void_p - kernel32.K32GetProcessMemoryInfo.restype = ctypes.c_int32 - kernel32.K32GetProcessMemoryInfo.argtypes = ( - ctypes.c_void_p, ctypes.POINTER(_Counters), ctypes.c_uint32) - counters = _Counters() - counters.cb = ctypes.sizeof(counters) - if not kernel32.K32GetProcessMemoryInfo( - kernel32.GetCurrentProcess(), ctypes.byref(counters), counters.cb): - raise ctypes.WinError() - return counters.peak_working_set -try: - import resource -except ImportError: - _peak = _windows_peak -else: - def _peak(): - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss -"; +class Shadowed(object): + def __init__(self, n: int) -> None: + self.n = n + self.extra = 1 -#[test] -fn an_unboxed_array_does_not_leak_its_buffer() { - let Some((python, toolchain)) = environment() else { - return; - }; - let dir = std::env::temp_dir().join("by_diff_arrayleak"); - let _ = std::fs::remove_dir_all(&dir); - // the buffer is `PyMem_Malloc`, not a `PyObject` — so a leak of one is invisible - // to `gc.get_objects()` and to a refcount check. the process's own footprint is - // the only thing that sees it - let source = "\ -def churn(rounds: int) -> float: - out = 0.0 - r = 0 - while r < rounds: - xs = [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] - out = out + xs[0] - r = r + 1 - return out -"; - if build_source( - source, - "by_diff_arrayleak", - &toolchain, - &dir, - &Options::default(), - ) - .is_err() - { - eprintln!("skipping: no working C toolchain"); - return; - } - let out = run( - &python, - &dir, - &format!( - "import by_diff_arrayleak as m\n{PEAK_FOOTPRINT}\ - m.churn(1000)\n\ - before = _peak()\n\ - m.churn(400000)\n\ - after = _peak()\n\ - print(after - before < before // 2 or after - before < 4_000_000)\n" - ), + def total(self) -> int: + return self.base() + self.extra +", + &[ + "m.Shadowed(3).total()", + "m.object(5).base()", + "isinstance(m.Shadowed(1), m.object)", + ], ); - assert_eq!(out, "True", "the array buffer leaks"); } #[test] -fn container_fast_paths_agree_and_respect_subclasses() { - // the fast paths are guarded on the *exact* type: a subclass may override - // `__getitem__` or `__missing__`, and a fast path that ignored that would be a - // wrong answer rather than a fast one - agree( - "containerprim", +fn a_parameter_defaulting_to_none_is_not_a_none_place() { + // `def f(x=None)` infers `Unknown | None`, and the gradual member is assignable to + // whatever is asked — so the union tested as assignable to `None` and the + // parameter got the `None` representation, which nothing else could be stored in. + // it is one of python's most common shapes + agree_python( + "nonedefault", "\ -def indexed(xs: list[int], i: int) -> int: - return xs[i] +def opened(f, mode=None): + if mode is None: + mode = 'rb' + return mode -def written(xs: list[int], i: int, v: int) -> str: - xs[i] = v - return str(xs) -def looked_up(d: dict[str, int], k: str) -> int: - return d[k] +def counted(items, start=None): + if start is None: + start = 0 + return start + len(items) -def sized(xs: object) -> int: - return len(xs) -def tupled(t: tuple[int, int], i: int) -> object: - return t[i] +def collected(seed=None): + if seed is None: + seed = [] + seed.append(1) + return seed ", &[ - "m.indexed([1, 2, 3], 1)", - "m.indexed([1, 2, 3], -1)", - "m.written([1, 2, 3], 0, 9)", - "m.looked_up({'a': 1}, 'a')", - "m.sized([1, 2, 3])", - "m.sized({'a': 1})", - "m.sized('abc')", - "m.tupled((4, 5), 1)", - // every error is python's own, message and class included - "[(type(e).__name__, str(e)) for e in [_capture(m.indexed, [1], 5)]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.written, [1], 5, 0)]]", - "[(type(e).__name__, repr(str(e))) for e in [_capture(m.looked_up, {}, 'zz')]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.tupled, (1,), 9)]]", - // a *subclass* overriding the protocol must still be honoured - "m.indexed(type('L', (list,), {'__getitem__': lambda s, i: 99})([1, 2]), 0)", - "m.looked_up(type('D', (dict,), {'__missing__': lambda s, k: 77})(), 'nope')", - "m.sized(type('L', (list,), {'__len__': lambda s: 42})([1]))", + "m.opened('x')", + "m.opened('x', 'wb')", + "m.counted([1, 2, 3])", + "m.counted([1, 2, 3], 10)", + "m.collected()", + "m.collected([9])", + "[m.opened('f', mode) for mode in [None, 'rb', 'wb']]", ], ); -} - -#[test] -fn a_comprehension_over_range_agrees() { - // the statement form has had a counting loop all along; the comprehension drove - // the iteration protocol instead — a `range` object, an iterator, a `next` and - // an unbox *per element*, for a loop whose bounds are right there - agree_with_declines( - "comprange", - "\ -def squares(n: int) -> object: - return [i * i for i in range(n)] +} -def filtered(n: int) -> object: - return [i for i in range(n) if i % 3 == 0] +#[test] +fn an_unannotated_parameter_with_a_default_is_not_a_float() { + // a parameter with no annotation infers `Unknown | `, and a + // *gradual* element is assignable both ways to everything — so one of them used to + // answer for both halves of the promotion test, and any `Unknown | T` read as + // `int | float`. that gave the parameter a `double` representation and a string + // default, which the c compiler rejected outright + agree_python( + "graduallead", + "\ +def failed(values, errmsg='negative value'): + for x in values: + if x < 0: + raise ValueError(errmsg) + yield x -def from_two(n: int) -> object: - return [i for i in range(2, n)] -def nested(n: int) -> object: - return [i * j for i in range(n) for j in range(n)] +def tagged(n, tag='t'): + return tag + str(n) -def buffered(n: int) -> float: - xs = [i * 1.5 for i in range(n)] - out = 0.0 - for x in xs: - out = out + x - return out -def empty(n: int) -> float: - xs = [i * 1.5 for i in range(0)] - out = 0.0 - for x in xs: - out = out + x - return out +def numeric(x, scale=2.0): + return x * scale ", &[ - "m.squares(5)", - "m.filtered(10)", - "m.from_two(6)", - "m.nested(3)", - "m.squares(0)", - "m.from_two(1)", - "m.buffered(5)", - "m.empty(0)", - // a negative bound is an empty range, not a backwards one - "m.squares(-3)", + "list(m.failed([1, 2], 'bad'))", + "list(m.failed([1, 2]))", + "[(type(e).__name__, str(e)) for e in [_capture(lambda: list(m.failed([1, -1])))]]", + "m.tagged(3)", + "m.tagged(3, 'x')", + "m.numeric(3)", + "m.numeric(3, 4.0)", + "m.numeric(2.5)", ], ); } #[test] -fn a_loop_over_an_unboxed_array_agrees() { - // the shape the whole representation exists for: an `i64` counter, no iterator - // object, no null test per step, and no bounds check — the counter is the - // lowering's own, so it is in range by construction - agree_with_declines( - "arrayloop", +fn a_gradual_value_narrowed_on_both_arms_is_still_an_object() { + // narrowing a gradual value gives an *intersection* holding it, so a conditional + // over both arms is a union of two of those and the gradual part sits one level + // down. it widens the whole type all the same: `(Unknown & C) | (Unknown & ~C)` is + // assignable to `None`, and reading that as a proof gave the result a `None` + // representation whose boundary rejected every value that was not one — which is + // how `enum.py` compiled and then could not be imported + agree_python( + "narrowedgradual", "\ -def total(n: int) -> float: - xs = [1.5, 2.5, 3.5, 4.5] - out = 0.0 - for x in xs: - out = out + x - return out - -def empty(n: int) -> float: - xs = [1.0] - out = 0.0 - for x in xs: - out = out + x - return out - -def broken(n: int) -> float: - xs = [1.0, 2.0, 3.0] - out = 0.0 - for x in xs: - if x > 1.5: - break - out = out + x - else: - out = -1.0 - return out - -def skipped(n: int) -> float: - xs = [1.0, 2.0, 3.0] - out = 0.0 - for x in xs: - if x > 1.5: - continue - out = out + x - return out +def unwrap(value): + return value.__func__ if isinstance(value, staticmethod) else value -def exhausted(n: int) -> float: - xs = [1.0, 2.0] - out = 0.0 - for x in xs: - out = out + x - else: - out = out + 100.0 - return out -def flags(n: int) -> int: - bs = [True, False, True, True] - total = 0 - for b in bs: - if b: - total = total + 1 - return total +def widest(value): + return value if isinstance(value, int) else value ", &[ - "m.total(0)", - "m.empty(0)", - // `break` skips the `else`, `continue` does not - "m.broken(0)", - "m.skipped(0)", - "m.exhausted(0)", - "m.flags(0)", + "m.unwrap(3)", + "m.unwrap('a')", + "m.unwrap(len).__name__", + "m.unwrap(staticmethod(len)).__name__", + "[m.widest(x) for x in (0, 'x', 2.5, None)]", + "m.widest(len).__name__", ], ); } #[test] -fn a_list_that_escapes_keeps_being_a_list() { - // the buffer is an optimization, not a restriction: a name that leaves the - // function never earns one in the first place, so it compiles exactly as it did - // before the representation existed - agree( - "bufferescape", +fn a_class_that_stores_through_setattr_keeps_its_interpreted_definition() { + // an emitted instance is its layout and nothing else — there is no `__dict__` + // behind it — so an attribute `setattr` names as a value has nowhere to go. the + // class has to stay interpreted, and the compiled module has to use *that* class: + // laying it out anyway is how `enum.py` reached `no __dict__ for setting new + // attributes` and could not be imported + agree_python_with_declines( + "setattrstore", "\ -def returned(n: int) -> object: - xs = [1.0, 2.0] - return xs +class Registry: + def __init__(self): + self.count = 0 -def passed(n: int) -> int: - xs = [1.0, 2.0] - return len(sorted(xs)) + def install(self, name, fn): + setattr(self, name, fn) + self.count = self.count + 1 -def stored(n: int) -> object: - xs = [1.0, 2.0] - return [xs, xs] + def run(self, name): + return getattr(self, name)() -def kept(n: int) -> float: - xs = [1.0, 2.0] - return xs[0] + xs[1] -def looped(n: int) -> float: - xs = [1.0, 2.0] - out = 0.0 - for x in xs: - out = out + x - return out +def build(): + r = Registry() + r.install('greet', lambda: 'hello') + return r ", &[ - // these keep a real list, and a real list is what comes back - "m.returned(0)", - "type(m.returned(0)).__name__", - "m.passed(0)", - "m.stored(0)", - // and these earn the buffer, invisibly - "m.kept(0)", - "m.looped(0)", + "m.build().run('greet')", + "m.build().count", + "[m.build().greet(), m.Registry().count]", ], ); } #[test] -fn a_from_import_inside_a_body_agrees() { - // plain python, so the interpreted leg is the source itself: the transpiler's - // lazy-import polyfill resolves `from pkg import submodule` wrongly, which would - // otherwise break the leg this is measured against rather than the compiled one +fn a_subscript_by_a_string_key_agrees() { + // a `str` widened to an `object` for a lookup must stay widened: substituting the + // source back handed `GetItem` a `str` register, which is a `PyObject *` in c and + // so compiled — but is a different representation, and the verifier said so agree_python( - "fromimport", + "strkey", "\ -def one(n: int) -> str: - from math import sqrt - return str(round(sqrt(n), 3)) +def read(d: dict[str, int], k: str) -> int: + return d[k] -def several(s: str) -> str: - from os.path import basename, dirname - return dirname(s) + '|' + basename(s) -def aliased(n: int) -> str: - from math import sqrt as root - return str(round(root(n), 3)) +def literal(d: dict[str, int]) -> int: + return d['k'] -# `urllib.parse` is not an attribute of `urllib` until something imports it, so -# this is the fromlist doing its job rather than an attribute read -def submodule(s: str) -> str: - from urllib import parse - return parse.quote(s) -def dotted(s: str) -> str: - from urllib.parse import quote - return quote(s) +def written(d: dict[str, int], k: str, v: int) -> object: + d[k] = v + return d -# a name the module does not have is an `ImportError`, not the `AttributeError` a -# plain attribute read would give — a guarded lazy import rests on the difference -def absent() -> object: - from os import definitely_not_a_real_name - return definitely_not_a_real_name -def guarded() -> str: - try: - from os import definitely_not_a_real_name - except ImportError: - return 'caught' - return 'no' +def missing(d: dict[str, int], k: str) -> object: + return d.get(k) ", &[ - "m.one(2)", - "m.several('/a/b/c.txt')", - "m.aliased(2)", - "m.submodule('a b/c')", - "m.dotted('a b/c')", - "m.guarded()", - // the type, the message and every attribute an `except` clause reads - "(lambda e: (type(e).__name__, str(e), e.name, e.path, e.name_from))(_capture(m.absent))", - // a thousand imports must not accumulate references to the module. a - // leaked module is one object with a climbing refcount, which no - // object-count check can see - "_repeated(lambda: m.one(2), 1000)", - "_refdelta('math', lambda: m.one(2), 500)", - "_refdelta('urllib.parse', lambda: m.submodule('a b'), 500)", + "m.read({'k': 1}, 'k')", + "m.literal({'k': 2})", + "m.written({}, 'a', 1)", + "m.missing({'k': 3}, 'k')", + "m.missing({'k': 3}, 'nope')", + "str(_capture(lambda: m.read({}, 'absent')))", + "[m.read({'a': 1, 'b': 2}, k) for k in ['a', 'b']]", + // the write goes straight to `PyDict_SetItem` for an *exact* dict, so a + // subclass that overrides `__setitem__` is what says the guard holds + "m.written(type('D', (dict,), \ + {'__setitem__': lambda s, k, v: dict.__setitem__(s, k, v * 10)})(), 'a', 5)", ], ); } #[test] -fn the_remaining_statement_and_expression_forms_agree() { +fn a_class_on_a_base_outside_the_module_agrees() { + // the type is built on whatever the name resolves to at module init, so the base + // has to *be* the real one: `raise` on a class that silently got `object` instead + // is a `TypeError`, which is how the two earlier attempts at this were caught + // + // the class declares no layout of its own — `basicsize` 0, and the base allocates + // and frees — and that has to hold **transitively**, or a subclass of a subclass + // declares a size smaller than its base and python rejects the type outright agree( - "coverage", + "external_base", "\ -def imports(n: int) -> str: - import math - return str(round(math.sqrt(n), 3)) +import collections -def aliased(n: int) -> str: - import os.path as p - return str(p.basename('/a/b')) +from collections import UserList -def del_key(d: dict[str, int]) -> str: - del d['a'] - return str(sorted(d.items())) -def del_item(xs: list[int]) -> str: - del xs[0] - return str(xs) +class MyError(Exception): + def label(self) -> str: + return \"mine\" -def del_attr(o: object) -> str: - del o.gone - return str(hasattr(o, 'gone')) -def ellipsis(n: int) -> object: - return ... +class Narrower(MyError): + def label(self) -> str: + return \"narrower\" -def negated(s: object) -> object: - return -s -def inverted(s: object) -> object: - return ~s +class Deeper(Narrower): + pass -def sliced(xs: list[int]) -> str: - return str(xs[1:3]) + str(xs[::2]) + str(xs[2:]) + str(xs[:2]) + str(xs[::-1]) -def slice_assigned(xs: list[int]) -> str: - xs[1:3] = [9, 9, 9] - return str(xs) +class Dotted(collections.OrderedDict): + # a base written as an attribute is a name to look up and then an attribute to + # walk, which is the only difference from a bare one + def label(self) -> str: + return \"dotted\" -def slice_deleted(xs: list[int]) -> str: - del xs[1:3] - return str(xs) -def walrus(xs: list[int]) -> str: - if (n := len(xs)) > 2: - return 'big ' + str(n) - return 'small ' + str(n) +class Listy(UserList): + def doubled(self) -> int: + return len(self) * 2 -def genexp(xs: list[int]) -> str: - return str(sum(x * 2 for x in xs)) + str(any(x > 2 for x in xs)) + str(max(x for x in xs)) -def declared_global(n: int) -> int: - global _counter - return n -", - &[ - "m.imports(16)", - "m.aliased(0)", - "m.del_key({'a': 1, 'b': 2})", - "m.del_item([1, 2, 3])", - "m.ellipsis(0)", - "m.sliced([1, 2, 3, 4])", - "m.slice_assigned([1, 2, 3, 4])", - "m.slice_deleted([1, 2, 3, 4])", - "m.walrus([1, 2, 3])", - "m.walrus([1])", - "m.genexp([1, 2, 3])", - "m.declared_global(4)", - // the protocol forms, on a type that answers them - "m.negated(type('N', (), {'__neg__': lambda s: 'neg'})())", - "m.inverted(type('N', (), {'__invert__': lambda s: 'inv'})())", - "m.del_attr(type('A', (), {})() if False else __import__('types').SimpleNamespace(gone=1))", - // and the errors each raises - "[(type(e).__name__, str(e)) for e in [_capture(m.del_key, {})]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.del_item, [])]]", - "[(type(e).__name__, str(e)) for e in [_capture(m.negated, object())]]", - ], - ); -} +def raising(n: int) -> int: + if n < 0: + raise Narrower(\"negative\") + return n * 2 -#[test] -fn a_bad_first_argument_raises_rather_than_crashing() { - // the wrapper releases every argument local on the error path, so one whose - // declaration a `goto` skipped would be released while indeterminate. a wrong - // type in the *first* parameter is the reachable case: it jumps over the rest. - // - // only the exception *type* is compared: the boundary rejects a bad argument - // where the interpreted leg gets as far as the operation that uses it, so the - // two agree that it is a `TypeError` and not on where it was raised - agree( - "badarg", - "\ -def two(a: int, b: str) -> int: - return a + len(b) -def three(a: int, b: str, c: list[int]) -> int: - return a + len(b) + len(c) -", - &[ - "type(_capture(m.two, 'x', 'y')).__name__", - "type(_capture(m.two, 1, 2)).__name__", - "type(_capture(m.three, 'x', 'y', [1])).__name__", - "type(_capture(m.three, 1, 'y', 'z')).__name__", - "m.two(1, 'yy')", - "m.three(1, 'yy', [1, 2])", - ], - ); -} +def catching(n: int) -> str: + try: + return str(raising(n)) + except MyError as e: + return \"caught \" + str(e) + \" \" + e.label() -#[test] -fn a_plain_python_loop_closure_shares_its_binding() { - let Some((python, toolchain)) = environment() else { - return; - }; - // python's loop binding is shared by every iteration, so all three closures - // see the last value. basedpython's is per-iteration. the compiled half has to - // follow the *source* language, not a flag — a `.py` fallback is python - let source = "\ -def counters() -> list[object]: - out = [] - for i in range(3): - def get() -> int: - return i - out.append(get) - return [f() for f in out] -"; - let dir = std::env::temp_dir().join("by_diff_pyloop"); - let interpreted = std::env::temp_dir().join("by_diff_pyloop_i"); - let _ = std::fs::remove_dir_all(&dir); - let _ = std::fs::remove_dir_all(&interpreted); - std::fs::create_dir_all(&interpreted).expect("the directory is created"); - std::fs::write(interpreted.join("by_diff_pyloop.py"), source).expect("written"); - let options = Options { - language: by_irbuild::Language::Python, - ..Options::default() - }; - let built = match build_source(source, "by_diff_pyloop", &toolchain, &dir, &options) { - Ok(built) => built, - Err(error) => { - eprintln!("skipping: no working C toolchain ({error})"); - return; - } - }; - assert!(built.declined.is_empty(), "{:?}", built.declined); +def hierarchy() -> str: + parts = ( + issubclass(Narrower, MyError), + issubclass(Deeper, Exception), + isinstance(Deeper(\"d\"), MyError), + Deeper(\"d\").label(), + ) + return \",\".join(str(x) for x in parts) - let body = "import by_diff_pyloop as m\nprint(m.counters())\n"; - let compiled = run(&python, &dir, body); - assert_eq!(compiled, run(&python, &interpreted, body)); - assert!(compiled.contains("[2, 2, 2]"), "{compiled}"); -} -#[test] -fn dunder_methods_fill_their_type_slots() { - // a method table cannot fill a slot: `repr(x)` reads `tp_repr` and never looks - // the name up. so each of these has to work *both* ways — through the slot and - // as an ordinary method - agree_python( - "dunders", - "\ -class Bag: - def __init__(self, items: list[int]) -> None: - self.items = items +def uses_list() -> str: + it = Listy([1, 2, 3]) + it.append(4) + return str(it.doubled()) + \" \" + str(len(it)) + \" \" + str(list(it)) - def __repr__(self) -> str: - return 'Bag(' + str(self.items) + ')' - def __str__(self) -> str: - return 'bag of ' + str(len(self.items)) +class WithStorage(Exception): + # a field of its own: the base allocates the instance, so this one lives in room + # asked for *past* it, and the class supplies the dealloc, traverse and clear that + # the base cannot write for storage it does not know about + def __init__(self, extra: int) -> None: + self.extra = extra - def __len__(self) -> int: - return len(self.items) + def bumped(self) -> int: + return self.extra + 1 - def __bool__(self) -> bool: - return len(self.items) > 1 + +class TwoBases(ValueError, IndexError): + # more than one base, none of them ours: python works out the mro and which of them + # owns the layout, and this class declares none of its own + def label(self) -> str: + return \"two\" + + +def caught_by_either(which: int) -> str: + try: + raise TwoBases(\"both\") + except IndexError as e: + if which == 0: + return \"index \" + str(e) + raise ", &[ - "repr(m.Bag([1, 2, 3]))", - "str(m.Bag([1, 2, 3]))", - "len(m.Bag([1, 2, 3]))", - "bool(m.Bag([1, 2, 3]))", - "bool(m.Bag([1]))", - "len(m.Bag([]))", - "bool(m.Bag([]))", - "f'{m.Bag([1])}'", - "'{}'.format(m.Bag([1, 2]))", - "m.Bag([1, 2]).__repr__()", - "m.Bag([1, 2]).__len__()", - "[b for b in [m.Bag([]), m.Bag([1, 2])] if b]", + "[m.catching(n) for n in (3, -1)]", + "m.hierarchy()", + "m.uses_list()", + "[type(e).__name__ for e in [_capture(m.raising, -2)]]", + "m.WithStorage(5).extra", + "m.WithStorage(5).bumped()", + "[type(e).__name__ for e in [_capture(m.WithStorage, 7)]]", + "(m.WithStorage(1).args, str(m.WithStorage(1)))", + "(m.Deeper.__mro__[1].__name__, m.Listy.__mro__[1].__name__)", + "m.caught_by_either(0)", + "[c.__name__ for c in m.TwoBases.__mro__[1:3]]", + "(issubclass(m.TwoBases, ValueError), issubclass(m.TwoBases, IndexError))", + "m.TwoBases('x').label()", + "(m.Dotted().label(), m.Dotted.__mro__[1].__name__)", ], ); } -#[test] -fn calling_a_resumable_frame_hands_back_the_state_object() { - // whatever the annotation says the body produces, the *call* gives the state - // object and the iteration or the `await` turns it back. there are three - // signature maps — module-level, methods, nested — and this has to hold in all of - // them, or the caller assigns a `PyObject *` into the annotation's representation - agree_python( - "resumablecall", - "\ -from typing import Any +/// the family a class appending storage grows: `TokenList` keeps a field past a `list` +/// instance, and every class under it keeps that one and adds none +/// +/// this is the stdlib's own shape — `email._header_value_parser` writes thirty-seven of +/// them — and the whole point is that a subclass adding no field of its own appends +/// nothing: what it stores is what `TokenList` stores, at the offset `TokenList` laid it +/// out and through the descriptor `TokenList` published. `Restating` is the same class +/// written the other way round, assigning an attribute the base already keeps +const APPENDED_FAMILY: &str = "\ +class TokenList(list): + + token_type = None + def __init__(self, *args): + super().__init__(*args) + self.defects = [] -def flags(n): - i = 0 - while i < n: - yield i > 0 - i = i + 1 + def kind(self): + return self.token_type + def defect_count(self): + return len(self.defects) -async def doubled(n: int) -> int: - return n * 2 +class Plain(TokenList): + pass -class Holder: - def __init__(self, n: int) -> None: - self.n = n - def counted(self): - i = 0 - while i < self.n: - yield i == 0 - i = i + 1 +class Named(TokenList): + token_type = 'named' - def total(self) -> int: - out = 0 - for seen in self.counted(): - if seen: - out = out + 1 - return out +class Deeper(Named): + token_type = 'deeper' -def nested(n: int) -> int: - def inner(): - i = 0 - while i < n: - yield i < 1 - i = i + 1 - out = 0 - for seen in inner(): - if seen: - out = out + 1 - return out + def kind(self): + return 'deep:' + str(self.token_type) -def drained(n: int) -> int: - out = 0 - for seen in flags(n): - if seen: - out = out + 1 - return out -", +class Restating(TokenList): + def __init__(self, *args): + super().__init__(*args) + self.defects = ['restated'] +"; + +#[test] +fn a_subclass_that_appends_nothing_past_a_base_agrees() { + agree_python( + "appendnothing", + APPENDED_FAMILY, &[ - "list(m.flags(3))", - "m.drained(3)", - "m.drained(0)", - "_run(m.doubled(5))", - "m.Holder(4).total()", - "list(m.Holder(2).counted())", - "m.nested(3)", - "m.nested(0)", + "[(type(t).__name__, list(t), t.defects, t.kind(), t.defect_count())\n\ + \x20 for t in (m.TokenList([0]), m.Plain([1]), m.Named([2]), m.Deeper([3]),\n\ + \x20 m.Restating([4]))]", + // the base's field, written and read through a subclass instance + "[(p.defects.append('one'), p.defects, p.defect_count(),\n\ + \x20 (p.append(9), list(p))[1]) for p in [m.Plain([1, 2])]]", + "[c.__name__ for c in m.Deeper.__mro__]", + "(isinstance(m.Deeper([1]), m.TokenList), isinstance(m.Restating([1]), list),\n\ + \x20issubclass(m.Plain, m.TokenList))", + "(m.Plain.token_type, m.Named.token_type, m.Deeper.token_type)", + "(sorted(m.Plain([3, 1, 2])), m.Plain([1, 2]) == [1, 2], m.Plain([1]) + [2])", + // python's own subclass of one, built by the class statement rather than here + "[(list(s([5])), s([5]).defects, s([5]).kind())\n\ + \x20 for s in [type('Py', (m.Plain,), {'token_type': 'py'})]]", + // every instance holds a cycle through the appended field, so the collector + // has to be able to see it and the deallocation has to release it + "(len([t for t in [m.Deeper([i]) for i in range(200)]\n\ + \x20 if t.defects.append(t) is None]), __import__('gc').collect() >= 0)", ], ); } #[test] -fn a_class_of_only_methods_compiles() { - // no `__init__` is not the same as no *layout*: a class of methods has an empty - // one, which is as representable as any other. `object.__init__` is what rejects a - // call with arguments, and it names the class rather than a method it does not have +fn a_subclass_that_appends_nothing_is_the_compiled_type() { + // the behaviour above is answered identically by a class that fell back to its + // interpreted definition, so it cannot say which build answered. + // `method_descriptor` can: a real type of ours holds one where the interpreted + // class holds a plain function + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_appendnothing_t"); + let _ = std::fs::remove_dir_all(&dir); + let built = match build_source( + APPENDED_FAMILY, + "by_diff_appendnothing_t", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_appendnothing_t as m\n\ + print(type(m.TokenList.__dict__['kind']).__name__,\n\ + \x20 type(m.Deeper.__dict__['kind']).__name__)\n\ + print(m.TokenList.__basicsize__ == m.Plain.__basicsize__\n\ + \x20 == m.Named.__basicsize__ == m.Deeper.__basicsize__)\n\ + print(m.Deeper([1]).defects, m.Restating([1]).defects)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + True\n\ + [] ['restated']" + ); +} + +#[test] +fn a_subclass_with_no_storage_stands_on_a_base_that_declined_later() { + // a base is settled as one of ours while the layouts settle, and only the body being + // lowered can turn it down after that — here a `__new__`, which fills a type slot + // with no adapter. a class with no storage of its own does not need the base to have + // stayed one: what stands under the name at import is a class either way, so it is + // built on the *name*, which is the construction every class over an outside base + // already takes. // - // `Empty` is where the class is a *static* type, which nothing can build on, so it - // fills the pair of slots itself and raises through them. `Consts` is a base, so it - // is a heap type and leaves both to `object` — where the message carries the module, - // because a type built from a spec keeps its module in `tp_name`. that difference is - // the price of not publishing an `__init__` the source never wrote: one installed - // anyway stands between everything built on the class and the `object.__init__` it - // should have reached - agree_python( - "onlymethods", - "\ -class Helpers: - def doubled(self, n: int) -> int: - return n * 2 + // `method_descriptor` against `function` is what says which type answered: `Plain` + // and `Deeper` are compiled types standing on the interpreted `TokenList` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_lostbase"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class TokenList(list): - def summed(self, n: int) -> int: - total = 0 - i = 0 - while i < n: - total = total + i - i = i + 1 - return total + token_type = None + def __new__(cls, *args): + return super().__new__(cls, *args) -class Consts: - SCALE = 3 + def __init__(self, *args): + super().__init__(*args) + self.defects = [] - def scaled(self, n: int) -> int: - return n * Consts.SCALE + def kind(self): + return self.token_type -class Empty: - pass +class Plain(TokenList): + token_type = 'plain' + def side(self): + return 'side:' + str(self.token_type) -class Sub(Consts): - def twice(self, n: int) -> int: - return self.scaled(n) * 2 -", - &[ - "m.Helpers().doubled(21)", - "m.Helpers().summed(5)", - "m.Consts().scaled(4)", - "m.Consts.SCALE", - "m.Sub().twice(4)", - "isinstance(m.Sub(), m.Consts)", - "type(m.Empty()).__name__", - "type(_capture(lambda: m.Consts(1))).__name__", - "str(_capture(lambda: m.Consts(1))).rsplit('.', 1)[-1]", - "str(_capture(lambda: m.Empty(1, 2)))", - // and a class anything can be built on publishes no `__init__` the source - // did not write. a static type still does, because it has to fill the slot - // pair itself — but nothing can be built on one, so no mro reaches it - "'__init__' in vars(m.Consts)", - "[m.Helpers().doubled(n) for n in [0, 1, 2]]", - ], + +class Deeper(Plain): + token_type = 'deeper' +"; + let built = match build_source( + source, + "by_diff_lostbase", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![( + "TokenList", + "`__new__` fills a type slot with no adapter yet" + )] + ); + let out = run( + &python, + &dir, + "import by_diff_lostbase as m\n\ + print([c.__name__ for c in m.Deeper.__mro__])\n\ + print(type(m.Plain.__dict__['side']).__name__,\n\ + \x20 type(m.TokenList.__dict__['kind']).__name__)\n\ + print(list(m.Plain([1, 2])), m.Plain([1, 2]).defects, m.Plain([1]).side())\n\ + print(m.Deeper([3]).kind(), m.Deeper([3]).side(), isinstance(m.Deeper([3]), m.TokenList))\n", + ); + assert_eq!( + out, + "['Deeper', 'Plain', 'TokenList', 'list', 'object']\n\ + method_descriptor function\n\ + [1, 2] [] side:plain\n\ + deeper side:deeper True" ); } #[test] -fn an_explicit_object_base_is_no_base_at_all() { - // `class C(object)` is what `class C:` already is, so it lays out and compiles the - // same way — but only when `object` really is the builtin +fn a_base_this_module_emits_beside_one_it_does_not_agrees() { + // a class may hold both kinds of base at once. it takes its whole layout from + // outside — the base of ours in the list lays nothing out, so it asks for no room — + // and python works out the mro and which of the bases owns the instance. + // + // the outside base may own a real one: `dict`, `int` and `Exception` each decide the + // instance, and getting that wrong writes this class's idea of a layout over theirs agree_python( - "objectbase", + "mixed_bases", "\ -class Plain(object): - def __init__(self, n: int) -> None: - self.n = n +import codecs - def doubled(self) -> int: - return self.n * 2 +class Ours: + def side(self) -> str: + return \"ours\" -class Derived(Plain): - def __init__(self, n: int) -> None: - self.n = n - self.extra = 1 + def reset(self) -> object: + # `object`, because the outside base's `reset` answers `None` and a narrower + # annotation would make the difference a representation error rather than a value + return \"ours reset\" + + +class OursFirst(Ours, codecs.Codec): + def label(self) -> str: + return \"first\" + + +class OutsideFirst(codecs.StreamWriter, Ours): + # the outside base comes first, so *its* `reset` is what the mro reaches and ours is + # shadowed — a direct call on a receiver typed as `Ours` would answer with ours + def label(self) -> str: + return \"outside\" + + +class AsDict(dict, Ours): + def label(self) -> str: + return \"dict\" + + +class AsInt(int, Ours): + def label(self) -> str: + return \"int\" + + +class OurError(Exception): + def which(self) -> str: + return \"ourerror\" + + +class Diamond(OurError, ValueError): + # the shared ancestor is outside: `Exception` is above both legs, and the mro has to + # place it once and after both + def label(self) -> str: + return \"diamond\" + + +class Under(OursFirst): + # a class of this module's own built on the mixture: the layout chain runs through + # a class that has none, so this one declares none either + def label(self) -> str: + return \"under\" - def total(self) -> int: - return self.doubled() + self.extra -", - &[ - "m.Plain(3).doubled()", - "m.Derived(4).total()", - "isinstance(m.Derived(1), m.Plain)", - "[m.Plain(n).doubled() for n in [0, 1, 2]]", - "m.Derived(2).n", - ], - ); -} -#[test] -fn a_module_that_binds_object_itself_keeps_its_own() { - // the name is resolved, not matched: a class of this module's own called `object` - // is the base, and taking the builtin instead would give the subclass the wrong one - agree_python( - "shadowedobject", - "\ -class object: - def __init__(self, n: int) -> None: - self.n = n +def through_the_base(o: Ours) -> str: + return o.side() - def base(self) -> int: - return self.n +def resetting(o: Ours) -> object: + return o.reset() -class Shadowed(object): - def __init__(self, n: int) -> None: - self.n = n - self.extra = 1 - def total(self) -> int: - return self.base() + self.extra +def exactly(which: int) -> str: + if which == 0: + return OursFirst().side() + return OutsideFirst(None).side() ", &[ - "m.Shadowed(3).total()", - "m.object(5).base()", - "isinstance(m.Shadowed(1), m.object)", + "[c.__name__ for c in m.OursFirst.__mro__]", + "[c.__name__ for c in m.OutsideFirst.__mro__]", + "[c.__name__ for c in m.Diamond.__mro__]", + "(m.OursFirst.__base__.__name__, m.AsDict.__base__.__name__, m.AsInt.__base__.__name__)", + "(m.OursFirst().side(), m.OursFirst().label())", + "(m.OutsideFirst(None).side(), m.OutsideFirst(None).label())", + "[m.through_the_base(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None), m.Under())]", + "[m.exactly(n) for n in (0, 1)]", + "[m.resetting(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None))]", + "([c.__name__ for c in m.Under.__mro__], m.Under().label(), m.Under().side())", + // the outside base still owns the instance it always owned + "(sorted(m.AsDict(a=1, b=2).items()), m.AsDict().label())", + "(int(m.AsInt(7)) + 1, m.AsInt(7).label(), m.AsInt(7).side())", + "(m.Diamond('boom').args, str(m.Diamond('boom')), m.Diamond('x').which(), m.Diamond('x').label())", + "(isinstance(m.Diamond('x'), ValueError), isinstance(m.Diamond('x'), m.OurError))", + // an instance of the mixture carries whatever `__dict__` the outside base + // brought: a class of ours alone has none, and losing it here silently + // turned every attribute read on one into a walk off the end of the object + "(lambda o: (setattr(o, 'kept', 3), o.kept, o.__dict__))(m.OursFirst())", + // and a python subclass of the result, which is a class statement on it + "(lambda C: (C().side(), C().label(), [c.__name__ for c in C.__mro__]))\ + (type('Py', (m.OursFirst,), {'label': lambda self: 'py'}))", ], ); } #[test] -fn a_parameter_defaulting_to_none_is_not_a_none_place() { - // `def f(x=None)` infers `Unknown | None`, and the gradual member is assignable to - // whatever is asked — so the union tested as assignable to `None` and the - // parameter got the `None` representation, which nothing else could be stored in. - // it is one of python's most common shapes - agree_python( - "nonedefault", - "\ -def opened(f, mode=None): - if mode is None: - mode = 'rb' - return mode +fn a_base_beside_an_outside_one_is_built_by_calling_its_metaclass() { + // a type spec takes its whole instance shape from the one base python picks out of + // the list. where that is a class of ours the `__dict__` an outside base needs is + // dropped, and the type then claims a managed dict it has no room for — the first + // attribute read on an instance walks off the object and segfaults. calling the + // metaclass works the shape out from every base at once + // + // the descriptors say which build answered: a class that fell back to its + // interpreted definition would agree on every value above and say `function` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_mixedmeta"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import abc +import codecs -def counted(items, start=None): - if start is None: - start = 0 - return start + len(items) +class Ours: + def side(self) -> str: + return \"ours\" -def collected(seed=None): - if seed is None: - seed = [] - seed.append(1) - return seed -", - &[ - "m.opened('x')", - "m.opened('x', 'wb')", - "m.counted([1, 2, 3])", - "m.counted([1, 2, 3], 10)", - "m.collected()", - "m.collected([9])", - "[m.opened('f', mode) for mode in [None, 'rb', 'wb']]", - ], +class Mixed(Ours, codecs.Codec): + def label(self) -> str: + return \"mixed\" + + +class Metaclassed(Ours, abc.ABC): + # a base whose metaclass is not `type` rules the spec out on its own, and the + # keyword the header carries has nowhere to go in one either + def label(self) -> str: + return \"meta\" +"; + let built = match build_source( + source, + "by_diff_mixedmeta", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_mixedmeta as m\n\ + print(type(m.Mixed.label).__name__, type(m.Ours.side).__name__)\n\ + print(m.Mixed.__base__.__name__, type(m.Metaclassed).__name__)\n\ + o = m.Mixed()\n\ + o.kept = 4\n\ + print(o.side(), o.label(), o.kept, o.__dict__)\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + Ours ABCMeta\n\ + ours mixed 4 {'kept': 4}" ); } #[test] -fn an_unannotated_parameter_with_a_default_is_not_a_float() { - // a parameter with no annotation infers `Unknown | `, and a - // *gradual* element is assignable both ways to everything — so one of them used to - // answer for both halves of the promotion test, and any `Unknown | T` read as - // `int | float`. that gave the parameter a `double` representation and a string - // default, which the c compiler rejected outright - agree_python( - "graduallead", +fn a_class_built_through_its_metaclass_agrees() { + // `PyType_FromSpecWithBases` gives the type it builds `type` for a metaclass, so it + // cannot be handed a base with any other one, and `PyType_FromMetaclass` refuses a + // metaclass that overrides `__new__` — which `ABCMeta` does. so the type is built + // the way python builds one: by calling the metaclass with a namespace. + // + // the methods go *in* that namespace rather than onto the finished type, which is + // what fills the type slots — `type.__new__` runs the same fixup a class statement + // does, so `__repr__` becomes `tp_repr` with no adapter of ours + agree( + "metaclass", "\ -def failed(values, errmsg='negative value'): - for x in values: - if x < 0: - raise ValueError(errmsg) - yield x +from abc import ABC, ABCMeta -def tagged(n, tag='t'): - return tag + str(n) +class Node(ABC): + def kind(self) -> str: + return \"node\" + def __repr__(self) -> str: + return \"Node(\" + self.kind() + \")\" -def numeric(x, scale=2.0): - return x * scale -", - &[ - "list(m.failed([1, 2], 'bad'))", - "list(m.failed([1, 2]))", - "[(type(e).__name__, str(e)) for e in [_capture(lambda: list(m.failed([1, -1])))]]", - "m.tagged(3)", - "m.tagged(3, 'x')", - "m.numeric(3)", - "m.numeric(3, 4.0)", - "m.numeric(2.5)", - ], - ); -} + def __len__(self) -> int: + return 3 -#[test] -fn a_gradual_value_narrowed_on_both_arms_is_still_an_object() { - // narrowing a gradual value gives an *intersection* holding it, so a conditional - // over both arms is a union of two of those and the gradual part sits one level - // down. it widens the whole type all the same: `(Unknown & C) | (Unknown & ~C)` is - // assignable to `None`, and reading that as a proof gave the result a `None` - // representation whose boundary rejected every value that was not one — which is - // how `enum.py` compiled and then could not be imported - agree_python( - "narrowedgradual", - "\ -def unwrap(value): - return value.__func__ if isinstance(value, staticmethod) else value + +class Deeper(Node): + # a base of *ours* whose own base is not: the layout is nobody's, and the metaclass + # is inherited down the chain + def kind(self) -> str: + return \"deeper\" -def widest(value): - return value if isinstance(value, int) else value -", - &[ - "m.unwrap(3)", - "m.unwrap('a')", - "m.unwrap(len).__name__", - "m.unwrap(staticmethod(len)).__name__", - "[m.widest(x) for x in (0, 'x', 2.5, None)]", - "m.widest(len).__name__", - ], - ); -} +class Keyed(metaclass=ABCMeta): + # no bases at all, only the keyword — python supplies `(object,)` itself + def kind(self) -> str: + return \"keyed\" -#[test] -fn a_class_that_stores_through_setattr_keeps_its_interpreted_definition() { - // an emitted instance is its layout and nothing else — there is no `__dict__` - // behind it — so an attribute `setattr` names as a value has nowhere to go. the - // class has to stay interpreted, and the compiled module has to use *that* class: - // laying it out anyway is how `enum.py` reached `no __dict__ for setting new - // attributes` and could not be imported - agree_python_with_declines( - "setattrstore", - "\ -class Registry: - def __init__(self): - self.count = 0 - def install(self, name, fn): - setattr(self, name, fn) - self.count = self.count + 1 +class Checked(ABC): + # an `__init__` of its own that stores nothing: it has to reach `tp_init`, which on + # this construction is python's dispatcher reading the namespace entry + def __init__(self, value: int) -> None: + if value < 0: + raise ValueError(\"negative\") - def run(self, name): - return getattr(self, name)() + +class Equated(ABC): + # a class defining `__eq__` and not `__hash__` is unhashable. a spec has to be told + # that; `type.__new__` does it for itself, so this is the same answer reached two + # different ways depending on which construction the bases allowed + def __eq__(self, other: object) -> bool: + return isinstance(other, Equated) -def build(): - r = Registry() - r.install('greet', lambda: 'hello') - return r +def widened(value: object) -> str: + if isinstance(value, Node): + return \"node \" + value.kind() + return \"other\" ", &[ - "m.build().run('greet')", - "m.build().count", - "[m.build().greet(), m.Registry().count]", + // the metaclass is the one python would have worked out, not `type` + "(type(m.Node).__name__, type(m.Keyed).__name__, type(m.Deeper).__name__)", + // and the whole ancestry with it + "[c.__name__ for c in m.Node.__mro__]", + "[c.__name__ for c in m.Keyed.__mro__]", + "[c.__name__ for c in m.Deeper.__mro__]", + // an abstract base answers for it, which is the point of the metaclass + "(isinstance(m.Node(), __import__('abc').ABC), issubclass(m.Node, __import__('abc').ABC))", + "(isinstance(m.Deeper(), m.Node), issubclass(m.Deeper, m.Node))", + // `ABCMeta.__new__` ran, and ran over *this* class's namespace + "sorted(m.Node.__abstractmethods__)", + // a dunder written in the class body fills its slot + "repr(m.Node())", + "repr(m.Deeper())", + "len(m.Node())", + // and the ordinary methods still bind + "(m.Node().kind(), m.Deeper().kind(), m.Keyed().kind())", + "[m.widened(v) for v in (m.Node(), m.Deeper(), 1)]", + // a written `__init__` runs, and the errors around it are python's own + "(m.Checked(1) is not None, type(_capture(m.Checked, -1)).__name__)", + "type(_capture(m.Checked)).__name__", + // and `__eq__` without `__hash__` still takes the hash away + "(m.Equated() == m.Equated(), m.Equated() == 1)", + "type(_capture(hash, m.Equated())).__name__", + // the class reports the module it was written in, not `builtins` + "(m.Node.__module__, m.Keyed.__module__)", + "(m.Node.__name__, m.Node.__qualname__)", + // an imported base reaches this module as a lazy proxy, so `__mro_entries__` + // is what turns it into the class — and python records what was written + "repr(m.Node.__orig_bases__)", ], ); } -#[test] -fn a_subscript_by_a_string_key_agrees() { - // a `str` widened to an `object` for a lookup must stay widened: substituting the - // source back handed `GetItem` a `str` register, which is a `PyObject *` in c and - // so compiled — but is a different representation, and the verifier said so - agree_python( - "strkey", - "\ -def read(d: dict[str, int], k: str) -> int: - return d[k] - - -def literal(d: dict[str, int]) -> int: - return d['k'] +/// a class inside a package reports the whole dotted module it was written in +/// +/// the test above asks this of a top-level module, where the module's own name and +/// its name inside its package are the same string — so it cannot see the +/// difference. cpython reads a type's `__module__` off the front of its `tp_name` +/// and its `__name__` off the back, and a compiled class that carried only its +/// file's stem named a module `sys.modules` has nothing under. `dataclasses` looks +/// exactly that up (`sys.modules.get(cls.__module__).__dict__`), so a package +/// member with a dataclass in it failed to import at all +#[test] +fn a_class_in_a_package_reports_the_package_it_came_from() { + let Some((python, toolchain)) = environment() else { + return; + }; + let source = "\ +class Point: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + def total(self) -> int: + return self.x + self.y +"; + let base = std::env::temp_dir().join("by_diff_pkgmodule"); + let _ = std::fs::remove_dir_all(&base); + let compiled_root = base.join("c"); + let interpreted_root = base.join("i"); + let compiled = compiled_root.join("by_diff_pkg"); + let interpreted = interpreted_root.join("by_diff_pkg"); + for dir in [&compiled, &interpreted] { + std::fs::create_dir_all(dir).expect("the package directory is created"); + std::fs::write(dir.join("__init__.py"), "").expect("the package marker is written"); + } + std::fs::write(interpreted.join("member.py"), source) + .expect("the interpreted module is written"); -def written(d: dict[str, int], k: str, v: int) -> object: - d[k] = v - return d + // the *root* of the output tree, not the package directory: the build writes the + // artefact at the module's own place within the tree, so handing it the package + // directory would nest a second `by_diff_pkg` inside the first + let built = match build_source( + source, + "by_diff_pkg.member", + &toolchain, + &compiled_root, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let body = "\ +import sys\n\ +import by_diff_pkg.member as m\n\ +print(m.Point.__module__, m.Point.__qualname__, m.Point.__name__)\n\ +print(sys.modules[m.Point.__module__] is m)\n\ +print(m.Point(2, 3).total())\n"; + let compiled_out = run(&python, &compiled_root, body); + let interpreted_out = run(&python, &interpreted_root, body); + assert_eq!( + compiled_out, interpreted_out, + "compiled {compiled_out}, interpreted {interpreted_out}" + ); + assert_eq!( + compiled_out, "by_diff_pkg.member Point Point\nTrue\n5", + "the module python imported is the module the class names" + ); -def missing(d: dict[str, int], k: str) -> object: - return d.get(k) -", - &[ - "m.read({'k': 1}, 'k')", - "m.literal({'k': 2})", - "m.written({}, 'a', 1)", - "m.missing({'k': 3}, 'k')", - "m.missing({'k': 3}, 'nope')", - "str(_capture(lambda: m.read({}, 'absent')))", - "[m.read({'a': 1, 'b': 2}, k) for k in ['a', 'b']]", - // the write goes straight to `PyDict_SetItem` for an *exact* dict, so a - // subclass that overrides `__setitem__` is what says the guard holds - "m.written(type('D', (dict,), \ - {'__setitem__': lambda s, k, v: dict.__setitem__(s, k, v * 10)})(), 'a', 5)", - ], + // …and it really was the emitted type that answered. a class that fell back to + // its interpreted definition answers all of the above identically, and holds a + // plain function where a type of ours holds a descriptor + let descriptor = run( + &python, + &compiled_root, + "import by_diff_pkg.member as m\n\ + print(type(m.Point.__dict__['total']).__name__)\n", ); + assert_eq!(descriptor, "method_descriptor"); } #[test] -fn a_class_on_a_base_outside_the_module_agrees() { - // the type is built on whatever the name resolves to at module init, so the base - // has to *be* the real one: `raise` on a class that silently got `object` instead - // is a `TypeError`, which is how the two earlier attempts at this were caught - // - // the class declares no layout of its own — `basicsize` 0, and the base allocates - // and frees — and that has to hold **transitively**, or a subclass of a subclass - // declares a size smaller than its base and python rejects the type outright +fn a_class_keyword_reaches_init_subclass_agrees() { + // a keyword is the metaclass's business, and a type spec has nowhere to put one. + // the base is built at runtime because a class this module *emits* would have its + // layout laid out here, which the metaclass construction gives up agree( - "external_base", + "classkeyword", "\ -import collections - -from collections import UserList - - -class MyError(Exception): - def label(self) -> str: - return \"mine\" - - -class Narrower(MyError): - def label(self) -> str: - return \"narrower\" - - -class Deeper(Narrower): - pass - +from abc import ABCMeta -class Dotted(collections.OrderedDict): - # a base written as an attribute is a name to look up and then an attribute to - # walk, which is the only difference from a bare one - def label(self) -> str: - return \"dotted\" +def _tagged(cls: type, tag: str = \"none\", ready: bool = False, level: int = 0) -> None: + cls.tag = tag + cls.ready = ready + cls.level = level -class Listy(UserList): - def doubled(self) -> int: - return len(self) * 2 +Base = ABCMeta(\"Base\", (), {\"__init_subclass__\": classmethod(_tagged)}) -def raising(n: int) -> int: - if n < 0: - raise Narrower(\"negative\") - return n * 2 +class Alpha(Base, tag=\"alpha\"): + def kind(self) -> str: + return \"alpha\" -def catching(n: int) -> str: - try: - return str(raising(n)) - except MyError as e: - return \"caught \" + str(e) + \" \" + e.label() +class Beta(Base, tag=\"beta\", ready=True, level=2): + # a literal keyword as well as a name: both are evaluated where a class body would + # have evaluated them + def kind(self) -> str: + return \"beta\" -def hierarchy() -> str: - parts = ( - issubclass(Narrower, MyError), - issubclass(Deeper, Exception), - isinstance(Deeper(\"d\"), MyError), - Deeper(\"d\").label(), - ) - return \",\".join(str(x) for x in parts) +# the keyword's own text is arbitrary, and the C string form of it stopped at a NUL +class Gamma(Base, tag=\"a\\x00b\"): + def kind(self) -> str: + return \"gamma\" +", + &[ + // the keyword reached `__init_subclass__`, which the metaclass's `__new__` + // is what calls + "(m.Alpha.tag, m.Alpha.ready, m.Alpha.level)", + "(m.Beta.tag, m.Beta.ready, m.Beta.level)", + "ascii(m.Gamma.tag)", + "(m.Alpha().kind(), m.Beta().kind(), m.Gamma().kind())", + // and the metaclass itself came from the base + "(type(m.Alpha).__name__, type(m.Beta).__name__)", + "[c.__name__ for c in m.Beta.__mro__]", + "sorted(m.Beta.__abstractmethods__)", + ], + ); +} -def uses_list() -> str: - it = Listy([1, 2, 3]) - it.append(4) - return str(it.doubled()) + \" \" + str(len(it)) + \" \" + str(list(it)) +#[test] +fn which_build_answers_for_a_metaclass_class() { + // the two `agree` tests above cannot see this on their own: a class that quietly + // fell back to its interpreted definition answers *identically*, so both would pass + // on a compiler that built nothing at all. this one names the build — a compiled + // method is a descriptor, an interpreted one is a plain function. + // + // it is also where the restriction is pinned down. the construction through a + // metaclass hands back a type whose instance layout is the metaclass's answer, so a + // class appending fields to its base has nowhere to put them and stays interpreted; + // its fieldless sibling, on the same base, does not + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metafields"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABC, ABCMeta -class WithStorage(Exception): - # a field of its own: the base allocates the instance, so this one lives in room - # asked for *past* it, and the class supplies the dealloc, traverse and clear that - # the base cannot write for storage it does not know about - def __init__(self, extra: int) -> None: - self.extra = extra +class Fielded(ABC): + def __init__(self, value: int) -> None: + self.value = value - def bumped(self) -> int: - return self.extra + 1 + def doubled(self) -> int: + return self.value * 2 -class TwoBases(ValueError, IndexError): - # more than one base, none of them ours: python works out the mro and which of them - # owns the layout, and this class declares none of its own +class Fieldless(ABC): def label(self) -> str: - return \"two\" + return \"fieldless\" -def caught_by_either(which: int) -> str: - try: - raise TwoBases(\"both\") - except IndexError as e: - if which == 0: - return \"index \" + str(e) - raise -", - &[ - "[m.catching(n) for n in (3, -1)]", - "m.hierarchy()", - "m.uses_list()", - "[type(e).__name__ for e in [_capture(m.raising, -2)]]", - "m.WithStorage(5).extra", - "m.WithStorage(5).bumped()", - "[type(e).__name__ for e in [_capture(m.WithStorage, 7)]]", - "(m.WithStorage(1).args, str(m.WithStorage(1)))", - "(m.Deeper.__mro__[1].__name__, m.Listy.__mro__[1].__name__)", - "m.caught_by_either(0)", - "[c.__name__ for c in m.TwoBases.__mro__[1:3]]", - "(issubclass(m.TwoBases, ValueError), issubclass(m.TwoBases, IndexError))", - "m.TwoBases('x').label()", - "(m.Dotted().label(), m.Dotted.__mro__[1].__name__)", - ], +class Keyed(metaclass=ABCMeta): + def label(self) -> str: + return \"keyed\" +"; + let built = match build_source( + source, + "by_diff_metafields", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![( + "Fielded", + "a class with fields of its own needs a base whose metaclass is `type`" + )] + ); + let out = run( + &python, + &dir, + "import by_diff_metafields as m\n\ + print(m.Fielded(4).doubled(), m.Fielded(4).value)\n\ + print(m.Fieldless().label(), m.Keyed().label())\n\ + print(type(m.Fielded).__name__, type(m.Fieldless).__name__, type(m.Keyed).__name__)\n\ + print(isinstance(m.Fielded(1), m.Fielded), isinstance(m.Fieldless(), m.Fieldless))\n\ + print(type(m.Fielded.doubled).__name__, type(m.Fieldless.label).__name__,\n\ + \x20 type(m.Keyed.label).__name__)\n", + ); + assert_eq!( + out, + "8 4\n\ + fieldless keyed\n\ + ABCMeta ABCMeta ABCMeta\n\ + True True\n\ + function method_descriptor method_descriptor" ); } #[test] -fn a_base_this_module_emits_beside_one_it_does_not_agrees() { - // a class may hold both kinds of base at once. it takes its whole layout from - // outside — the base of ours in the list lays nothing out, so it asks for no room — - // and python works out the mro and which of the bases owns the instance. +fn a_metaclass_that_remakes_a_class_level_constant_is_turned_down_after_the_call() { + // the constants go into the namespace the metaclass is handed, which is enough for a + // metaclass that only *reads* one. an `EnumType` does not read them: it builds a + // *member* out of `STRICT = auto()`, and the member is not the value the module body + // already took a reference to. `FIRST is Boundary.STRICT` is what that costs, and no + // amount of writing the namespace fixes it. // - // the outside base may own a real one: `dict`, `int` and `Exception` each decide the - // instance, and getting that wrong writes this class's idea of a layout over theirs - agree_python( - "mixed_bases", - "\ -import codecs + // so the class is asked afterwards whether it kept what it was handed, and where it + // did not the interpreted definition stands. the two classes are the boundary: same + // base, same metaclass, and only `Boundary` has a constant. `function` against + // `method_descriptor` is what says the refusal is exactly that narrow — a guard that + // never fired would make `Boundary` say `method_descriptor` and lose `FIRST` + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaconstant"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from enum import StrEnum, auto -class Ours: - def side(self) -> str: - return \"ours\" +class Boundary(StrEnum): + STRICT = auto() + CONFORM = auto() - def reset(self) -> object: - # `object`, because the outside base's `reset` answers `None` and a narrower - # annotation would make the difference a representation error rather than a value - return \"ours reset\" + def shout(self) -> str: + return self.name -class OursFirst(Ours, codecs.Codec): - def label(self) -> str: - return \"first\" +class Plain(StrEnum): + # no constant, so the namespace the metaclass is handed is the whole class body + def shout(self) -> str: + return \"plain\" -class OutsideFirst(codecs.StreamWriter, Ours): - # the outside base comes first, so *its* `reset` is what the mro reaches and ours is - # shadowed — a direct call on a receiver typed as `Ours` would answer with ours - def label(self) -> str: - return \"outside\" +FIRST = Boundary.STRICT +"; + let built = match build_source( + source, + "by_diff_metaconstant", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // nothing is declined: the class is emitted, and only the *construction* the runtime + // picks for it changes + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaconstant as m\n\ + print(m.Boundary._member_names_)\n\ + print(isinstance(m.FIRST, m.Boundary), m.FIRST is m.Boundary.STRICT)\n\ + print(m.Boundary.STRICT.shout(), m.Boundary('strict') is m.Boundary.STRICT)\n\ + print([c.__name__ for c in m.Boundary.__mro__])\n\ + print(type(m.Boundary).__name__, type(m.Plain).__name__)\n\ + print(m.Plain._member_names_, m.Plain().shout() if False else 'plain')\n\ + print(type(m.Boundary.shout).__name__, type(m.Plain.shout).__name__)\n", + ); + assert_eq!( + out, + "['STRICT', 'CONFORM']\n\ + True True\n\ + STRICT True\n\ + ['Boundary', 'StrEnum', 'str', 'ReprEnum', 'Enum', 'object']\n\ + EnumType EnumType\n\ + [] plain\n\ + function method_descriptor" + ); +} +#[test] +fn a_dunder_the_module_body_hangs_on_a_class_keeps_it_off_the_compiled_surface() { + // `ctypes` writes `c_byte.__ctype_le__ = c_byte.__ctype_be__ = c_byte` under the class + // statement, and the adoption that carries a twin's attributes across leaves every + // dunder behind — a dunder is what a type slot answers, and a second answer sitting in + // the dict would disagree with it. so the attribute has nowhere to land and the class + // has to decline. + // + // the two classes are the two constructions: `Meta` can only be built by calling its + // metaclass and `Spec` comes from a type spec, and the adoption is the same for both. + // `plain` is the boundary in the other direction — a name that is not a dunder *is* + // carried, and is not a reason to turn anything down + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_hungdunder"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABCMeta -class AsDict(dict, Ours): - def label(self) -> str: - return \"dict\" +class Meta(metaclass=ABCMeta): + TAG = 1 -class AsInt(int, Ours): def label(self) -> str: - return \"int\" - - -class OurError(Exception): - def which(self) -> str: - return \"ourerror\" - + return \"meta\" -class Diamond(OurError, ValueError): - # the shared ancestor is outside: `Exception` is above both legs, and the mro has to - # place it once and after both - def label(self) -> str: - return \"diamond\" +class Spec: + TAG = 2 -class Under(OursFirst): - # a class of this module's own built on the mixture: the layout chain runs through - # a class that has none, so this one declares none either def label(self) -> str: - return \"under\" + return \"spec\" -def through_the_base(o: Ours) -> str: - return o.side() +class Untouched: + TAG = 3 - -def resetting(o: Ours) -> object: - return o.reset() + def label(self) -> str: + return \"untouched\" -def exactly(which: int) -> str: - if which == 0: - return OursFirst().side() - return OutsideFirst(None).side() -", - &[ - "[c.__name__ for c in m.OursFirst.__mro__]", - "[c.__name__ for c in m.OutsideFirst.__mro__]", - "[c.__name__ for c in m.Diamond.__mro__]", - "(m.OursFirst.__base__.__name__, m.AsDict.__base__.__name__, m.AsInt.__base__.__name__)", - "(m.OursFirst().side(), m.OursFirst().label())", - "(m.OutsideFirst(None).side(), m.OutsideFirst(None).label())", - "[m.through_the_base(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None), m.Under())]", - "[m.exactly(n) for n in (0, 1)]", - "[m.resetting(o) for o in (m.Ours(), m.OursFirst(), m.OutsideFirst(None))]", - "([c.__name__ for c in m.Under.__mro__], m.Under().label(), m.Under().side())", - // the outside base still owns the instance it always owned - "(sorted(m.AsDict(a=1, b=2).items()), m.AsDict().label())", - "(int(m.AsInt(7)) + 1, m.AsInt(7).label(), m.AsInt(7).side())", - "(m.Diamond('boom').args, str(m.Diamond('boom')), m.Diamond('x').which(), m.Diamond('x').label())", - "(isinstance(m.Diamond('x'), ValueError), isinstance(m.Diamond('x'), m.OurError))", - // an instance of the mixture carries whatever `__dict__` the outside base - // brought: a class of ours alone has none, and losing it here silently - // turned every attribute read on one into a walk off the end of the object - "(lambda o: (setattr(o, 'kept', 3), o.kept, o.__dict__))(m.OursFirst())", - // and a python subclass of the result, which is a class statement on it - "(lambda C: (C().side(), C().label(), [c.__name__ for c in C.__mro__]))\ - (type('Py', (m.OursFirst,), {'label': lambda self: 'py'}))", - ], +Meta.__marker__ = Meta +Spec.__marker__ = Spec +Untouched.plain = 4 +"; + let built = match build_source( + source, + "by_diff_hungdunder", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "Meta", + "the module body writes `__marker__` onto `Meta`, which the emitted type does not carry" + ), + ( + "Spec", + "the module body writes `__marker__` onto `Spec`, which the emitted type does not carry" + ) + ] + ); + let out = run( + &python, + &dir, + "import by_diff_hungdunder as m\n\ + print(m.Meta.__marker__ is m.Meta, m.Spec.__marker__ is m.Spec, m.Untouched.plain)\n\ + print(m.Meta().label(), m.Spec().label(), m.Untouched().label())\n\ + print(type(m.Meta.label).__name__, type(m.Spec.label).__name__,\n\ + \x20 type(m.Untouched.label).__name__)\n", + ); + assert_eq!( + out, + "True True 4\n\ + meta spec untouched\n\ + function function method_descriptor" ); } #[test] -fn a_base_beside_an_outside_one_is_built_by_calling_its_metaclass() { - // a type spec takes its whole instance shape from the one base python picks out of - // the list. where that is a class of ours the `__dict__` an outside base needs is - // dropped, and the type then claims a managed dict it has no room for — the first - // attribute read on an instance walks off the object and segfaults. calling the - // metaclass works the shape out from every base at once +fn a_constant_that_reads_back_differently_every_time_is_not_turned_down_for_it() { + // a class-level constant is read out of the *mapping* the class body wrote rather + // than through a lookup on the class, and `__class_getitem__ = classmethod(f)` is + // where the two differ: a lookup runs the descriptor and answers a freshly bound + // method, and one bound to the interpreted definition at that. copying that is what + // used to make `Holder[int]` answer the twin instead of `Holder`, and it made the + // check after the metaclass call — which compares the finished class against the + // values it was handed — turn down every class with such a constant, since no two + // reads are the same object. almost every container type in the stdlib has one. // - // the descriptors say which build answered: a class that fell back to its - // interpreted definition would agree on every value above and say `function` + // `Spec` is the boundary: both constructions carry the classmethod itself now, so + // both answer the class the interpreter answers let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_mixedmeta"); + let dir = std::env::temp_dir().join("by_diff_metaunstable"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -import abc -import codecs +from abc import ABCMeta -class Ours: - def side(self) -> str: - return \"ours\" +def _get(cls, item): + return cls -class Mixed(Ours, codecs.Codec): +class Holder(metaclass=ABCMeta): + __class_getitem__ = classmethod(_get) + def label(self) -> str: - return \"mixed\" + return \"holder\" -class Metaclassed(Ours, abc.ABC): - # a base whose metaclass is not `type` rules the spec out on its own, and the - # keyword the header carries has nowhere to go in one either +class Spec: + __class_getitem__ = classmethod(_get) + def label(self) -> str: - return \"meta\" + return \"spec\" "; let built = match build_source( source, - "by_diff_mixedmeta", + "by_diff_metaunstable", &toolchain, &dir, &Options { @@ -6538,209 +8746,282 @@ class Metaclassed(Ours, abc.ABC): let out = run( &python, &dir, - "import by_diff_mixedmeta as m\n\ - print(type(m.Mixed.label).__name__, type(m.Ours.side).__name__)\n\ - print(m.Mixed.__base__.__name__, type(m.Metaclassed).__name__)\n\ - o = m.Mixed()\n\ - o.kept = 4\n\ - print(o.side(), o.label(), o.kept, o.__dict__)\n", + "import by_diff_metaunstable as m\n\ + print(m.Holder().label(), m.Spec().label())\n\ + print(type(m.Holder.label).__name__, type(m.Spec.label).__name__)\n\ + print(m.Holder[int] is m.Holder, m.Spec[int] is m.Spec)\n\ + print(m.Holder[int] is m.Spec[int])\n", ); assert_eq!( out, - "method_descriptor method_descriptor\n\ - Ours ABCMeta\n\ - ours mixed 4 {'kept': 4}" + "holder spec\n\ + method_descriptor method_descriptor\n\ + True True\n\ + False" ); } #[test] -fn a_class_built_through_its_metaclass_agrees() { - // `PyType_FromSpecWithBases` gives the type it builds `type` for a metaclass, so it - // cannot be handed a base with any other one, and `PyType_FromMetaclass` refuses a - // metaclass that overrides `__new__` — which `ABCMeta` does. so the type is built - // the way python builds one: by calling the metaclass with a namespace. +fn a_metaclass_that_raises_on_the_namespace_it_is_handed_leaves_the_import_standing() { + // `ssl.Purpose`'s shape, and the one thing worse than a wrong answer: `EnumType` is + // handed a namespace whose members are the twin's *finished* ones and tries to build a + // member out of a member — `Obj.__new__(cls, a, b)` against a `__new__` that takes one + // argument. that raises before the check after the call could turn the class down, and + // propagating it fails `by_exec` and takes the whole import with it. // - // the methods go *in* that namespace rather than onto the finished type, which is - // what fills the type slots — `type.__new__` runs the same fixup a class statement - // does, so `__repr__` becomes `tp_repr` with no adapter of ours - agree( - "metaclass", - "\ -from abc import ABC, ABCMeta - - -class Node(ABC): - def kind(self) -> str: - return \"node\" - - def __repr__(self) -> str: - return \"Node(\" + self.kind() + \")\" - - def __len__(self) -> int: - return 3 - + // the interpreted definition already built this class — the fallback source ran first — + // so the raise says the reconstruction is wrong, not that the class is unbuildable. it + // is the same refusal the check makes, reached earlier. `Plain` is the boundary: the + // same metaclass with a namespace it can work with still reaches the compiled type + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaraise"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from collections import namedtuple +from enum import Enum -class Deeper(Node): - # a base of *ours* whose own base is not: the layout is nobody's, and the metaclass - # is inherited down the chain - def kind(self) -> str: - return \"deeper\" +class Obj(namedtuple(\"Obj\", \"a b\")): + __slots__ = () -class Keyed(metaclass=ABCMeta): - # no bases at all, only the keyword — python supplies `(object,)` itself - def kind(self) -> str: - return \"keyed\" + def __new__(cls, text): + return super().__new__(cls, text, text.upper()) -class Checked(ABC): - # an `__init__` of its own that stores nothing: it has to reach `tp_init`, which on - # this construction is python's dispatcher reading the namespace entry - def __init__(self, value: int) -> None: - if value < 0: - raise ValueError(\"negative\") +class Kind(Obj, Enum): + ONE = \"one\" + TWO = \"two\" -class Equated(ABC): - # a class defining `__eq__` and not `__hash__` is unhashable. a spec has to be told - # that; `type.__new__` does it for itself, so this is the same answer reached two - # different ways depending on which construction the bases allowed - def __eq__(self, other: object) -> bool: - return isinstance(other, Equated) +class Plain(Enum): + def shout(self) -> str: + return \"plain\" -def widened(value: object) -> str: - if isinstance(value, Node): - return \"node \" + value.kind() - return \"other\" -", - &[ - // the metaclass is the one python would have worked out, not `type` - "(type(m.Node).__name__, type(m.Keyed).__name__, type(m.Deeper).__name__)", - // and the whole ancestry with it - "[c.__name__ for c in m.Node.__mro__]", - "[c.__name__ for c in m.Keyed.__mro__]", - "[c.__name__ for c in m.Deeper.__mro__]", - // an abstract base answers for it, which is the point of the metaclass - "(isinstance(m.Node(), __import__('abc').ABC), issubclass(m.Node, __import__('abc').ABC))", - "(isinstance(m.Deeper(), m.Node), issubclass(m.Deeper, m.Node))", - // `ABCMeta.__new__` ran, and ran over *this* class's namespace - "sorted(m.Node.__abstractmethods__)", - // a dunder written in the class body fills its slot - "repr(m.Node())", - "repr(m.Deeper())", - "len(m.Node())", - // and the ordinary methods still bind - "(m.Node().kind(), m.Deeper().kind(), m.Keyed().kind())", - "[m.widened(v) for v in (m.Node(), m.Deeper(), 1)]", - // a written `__init__` runs, and the errors around it are python's own - "(m.Checked(1) is not None, type(_capture(m.Checked, -1)).__name__)", - "type(_capture(m.Checked)).__name__", - // and `__eq__` without `__hash__` still takes the hash away - "(m.Equated() == m.Equated(), m.Equated() == 1)", - "type(_capture(hash, m.Equated())).__name__", - // the class reports the module it was written in, not `builtins` - "(m.Node.__module__, m.Keyed.__module__)", - "(m.Node.__name__, m.Node.__qualname__)", - // an imported base reaches this module as a lazy proxy, so `__mro_entries__` - // is what turns it into the class — and python records what was written - "repr(m.Node.__orig_bases__)", - ], +FIRST = Kind.ONE +"; + let built = match build_source( + source, + "by_diff_metaraise", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + // `Obj` names a call as its base, which is a decline of its own and not this one + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + ["Obj"] + ); + let out = run( + &python, + &dir, + "import by_diff_metaraise as m\n\ + print(m.Kind.ONE.a, m.Kind.ONE.b, m.FIRST is m.Kind.ONE)\n\ + print(isinstance(m.FIRST, m.Kind), m.Kind(m.Kind.ONE.value) is m.FIRST)\n\ + print(m.Kind._member_names_, type(m.Kind).__name__)\n\ + print(type(m.Plain.shout).__name__)\n", + ); + assert_eq!( + out, + "one ONE True\n\ + True True\n\ + ['ONE', 'TWO'] EnumType\n\ + method_descriptor" ); } #[test] -fn a_class_keyword_reaches_init_subclass_agrees() { - // a keyword is the metaclass's business, and a type spec has nowhere to put one. - // the base is built at runtime because a class this module *emits* would have its - // layout laid out here, which the metaclass construction gives up - agree( - "classkeyword", - "\ -from abc import ABCMeta - +fn a_class_level_constant_beside_a_base_of_ours_reaches_the_metaclass_namespace() { + // the shape 69 of the stdlib's 84 instances of the old constant decline had: no class + // keyword anywhere, a base this module emits standing beside one from outside, and a + // constant. a spec cannot work that base list out, so the metaclass builds it — and + // the constant goes into the namespace it is handed rather than onto the type + // afterwards, which is what makes that construction answer for such a class at all. + // + // `issubclass` is what the decline used to be protecting: with `Reader` interpreted + // and `Codec` emitted it answers False where python answers True, so the two had to + // decline together. both are compiled here, and `method_descriptor` against + // `function` is what says so — `Alone` is the boundary that never needed the base + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaconstant_base"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import codecs -def _tagged(cls: type, tag: str = \"none\", ready: bool = False, level: int = 0) -> None: - cls.tag = tag - cls.ready = ready - cls.level = level +class Codec(codecs.Codec): + def label(self) -> str: + return \"codec\" -Base = ABCMeta(\"Base\", (), {\"__init_subclass__\": classmethod(_tagged)}) +class Reader(Codec, codecs.StreamReader): + tag = 1 -class Alpha(Base, tag=\"alpha\"): def kind(self) -> str: - return \"alpha\" + return \"reader\" -class Beta(Base, tag=\"beta\", ready=True, level=2): - # a literal keyword as well as a name: both are evaluated where a class body would - # have evaluated them +class Alone(codecs.Codec): + tag = 2 + def kind(self) -> str: - return \"beta\" + return \"alone\" +"; + let built = match build_source( + source, + "by_diff_metaconstant_base", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!(declined, Vec::new()); + let out = run( + &python, + &dir, + "import by_diff_metaconstant_base as m\n\ + print(issubclass(m.Reader, m.Codec), m.Reader.tag, m.Alone.tag)\n\ + print([c.__name__ for c in m.Reader.__mro__])\n\ + print(type(m.Codec.label).__name__, type(m.Reader.kind).__name__,\n\ + \x20 type(m.Alone.kind).__name__)\n", + ); + assert_eq!( + out, + "True 1 2\n\ + ['Reader', 'Codec', 'StreamReader', 'Codec', 'object']\n\ + method_descriptor method_descriptor method_descriptor" + ); +} +#[test] +fn a_slots_declaration_reaches_the_metaclass_rather_than_the_finished_type() { + // `__slots__` is the constant that proves the namespace is where these have to go. + // `type.__new__` reads it *out of the namespace* to decide whether the instances get + // a dict at all, so one copied onto the finished type afterwards is not a `__slots__` + // — the class already has the dict, and the entry sits there saying otherwise. + // + // 29 stdlib classes are this shape. `Open` is the boundary: the same base and the + // same construction with no `__slots__`, and python gives *its* instances a dict, so + // this is not a rule about emitted classes but about what the body wrote + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_metaslots"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +from abc import ABC -# the keyword's own text is arbitrary, and the C string form of it stopped at a NUL -class Gamma(Base, tag=\"a\\x00b\"): - def kind(self) -> str: - return \"gamma\" -", - &[ - // the keyword reached `__init_subclass__`, which the metaclass's `__new__` - // is what calls - "(m.Alpha.tag, m.Alpha.ready, m.Alpha.level)", - "(m.Beta.tag, m.Beta.ready, m.Beta.level)", - "ascii(m.Gamma.tag)", - "(m.Alpha().kind(), m.Beta().kind(), m.Gamma().kind())", - // and the metaclass itself came from the base - "(type(m.Alpha).__name__, type(m.Beta).__name__)", - "[c.__name__ for c in m.Beta.__mro__]", - "sorted(m.Beta.__abstractmethods__)", - ], + +class Slotted(ABC): + __slots__ = () + + def label(self) -> str: + return \"slotted\" + + +class Open(ABC): + def label(self) -> str: + return \"open\" +"; + let built = match build_source( + source, + "by_diff_metaslots", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_metaslots as m\n\ + print(m.Slotted.__slots__, hasattr(m.Slotted(), '__dict__'), hasattr(m.Open(), '__dict__'))\n\ + print(m.Slotted().label(), m.Open().label())\n\ + print(type(m.Slotted.label).__name__, type(m.Open.label).__name__)\n", + ); + assert_eq!( + out, + "() False True\n\ + slotted open\n\ + method_descriptor method_descriptor" ); } #[test] -fn which_build_answers_for_a_metaclass_class() { - // the two `agree` tests above cannot see this on their own: a class that quietly - // fell back to its interpreted definition answers *identically*, so both would pass - // on a compiler that built nothing at all. this one names the build — a compiled - // method is a descriptor, an interpreted one is a plain function. +fn a_class_constant_naming_another_class_reaches_the_metaclass_namespace_remapped() { + // the value a constant carries comes off the twin, so `pair = Other` in a class body + // hands over the *interpreted* `Other` — a class nothing else in the module can + // reach, and one `isinstance` denies against `m.Other`. the substitution that fixes + // that is the copy's, and the namespace has to make the same one or the two + // constructions disagree about what a constant is. // - // it is also where the restriction is pinned down. the construction through a - // metaclass hands back a type whose instance layout is the metaclass's answer, so a - // class appending fields to its base has nowhere to put them and stays interpreted; - // its fieldless sibling, on the same base, does not + // `Below` is what forces the metaclass here: `ABCMeta` closes the spec, so the + // constant goes in before the call rather than onto the type after it let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_metafields"); + let dir = std::env::temp_dir().join("by_diff_metaconstant_remap"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from abc import ABC, ABCMeta - - -class Fielded(ABC): - def __init__(self, value: int) -> None: - self.value = value +from abc import ABCMeta - def doubled(self) -> int: - return self.value * 2 +class Other: + def kind(self) -> str: + return \"other\" -class Fieldless(ABC): - def label(self) -> str: - return \"fieldless\" +class Below(metaclass=ABCMeta): + pair = Other -class Keyed(metaclass=ABCMeta): def label(self) -> str: - return \"keyed\" + return \"below\" "; let built = match build_source( source, - "by_diff_metafields", + "by_diff_metaconstant_remap", &toolchain, &dir, &Options { @@ -6755,80 +9036,61 @@ class Keyed(metaclass=ABCMeta): return; } }; - let declined: Vec<(&str, &str)> = built - .declined - .iter() - .map(|declined| (declined.name.as_str(), declined.reason.as_str())) - .collect(); - assert_eq!( - declined, - vec![( - "Fielded", - "a class with fields of its own needs a base whose metaclass is `type`" - )] - ); + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let out = run( &python, &dir, - "import by_diff_metafields as m\n\ - print(m.Fielded(4).doubled(), m.Fielded(4).value)\n\ - print(m.Fieldless().label(), m.Keyed().label())\n\ - print(type(m.Fielded).__name__, type(m.Fieldless).__name__, type(m.Keyed).__name__)\n\ - print(isinstance(m.Fielded(1), m.Fielded), isinstance(m.Fieldless(), m.Fieldless))\n\ - print(type(m.Fielded.doubled).__name__, type(m.Fieldless.label).__name__,\n\ - \x20 type(m.Keyed.label).__name__)\n", + "import by_diff_metaconstant_remap as m\n\ + print(m.Below.pair is m.Other, isinstance(m.Below.pair(), m.Other))\n\ + print(m.Below.pair().kind(), m.Below().label())\n\ + print(type(m.Below.label).__name__, type(m.Other.kind).__name__)\n", ); assert_eq!( out, - "8 4\n\ - fieldless keyed\n\ - ABCMeta ABCMeta ABCMeta\n\ - True True\n\ - function method_descriptor method_descriptor" + "True True\n\ + other below\n\ + method_descriptor method_descriptor" ); } #[test] -fn a_class_level_constant_keeps_its_class_off_the_metaclass_construction() { - // a class-level constant is copied onto the *finished* type, and through a metaclass - // that is too late: the metaclass has already decided what the class defines from a - // namespace the constant was never in. an `EnumType` handed a memberless namespace - // declares no members, and the copy then lands them in the type's dict behind its - // back — `Boundary.STRICT` answers while `_member_names_` is empty and - // `isinstance(FIRST, Boundary)` is False. that was a silent wrong answer, so a class - // with any constant keeps the interpreted definition instead. +fn a_class_the_module_pops_out_of_its_own_globals_stays_off_the_compiled_surface() { + // `ast` builds `Num` and then pops the name straight out of its own globals. that is + // a `del` whose target this cannot read — the name comes off a comprehension there — + // so every definition the module writes is treated as one the pop could have taken. + // installing a compiled `Gone` over a name the body removed would put a class on the + // surface python does not have there, and the interpreted definition the construction + // would otherwise fall back to is not there to be found either. // - // the two classes are the boundary: same base, same metaclass, and only `Boundary` - // has a constant. `function` against `method_descriptor` is what says the fallback is - // exactly that narrow — widening the gate would make `Plain` say `function` too + // the class-level-constant gate used to carry this, and this is what stayed behind + // when it went. `Kept` is no longer a boundary — the rule reaches the whole module, + // which is what its second decline says let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_metaconstant"); + let dir = std::env::temp_dir().join("by_diff_poppedclass"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -from enum import StrEnum, auto +from abc import ABCMeta -class Boundary(StrEnum): - STRICT = auto() - CONFORM = auto() +class Gone(metaclass=ABCMeta): + TAG = 1 - def shout(self) -> str: - return self.name + def label(self) -> str: + return \"gone\" -class Plain(StrEnum): - # no constant, so the namespace the metaclass is handed is the whole class body - def shout(self) -> str: - return \"plain\" +class Kept: + def label(self) -> str: + return \"kept\" -FIRST = Boundary.STRICT +HIDDEN = {name: globals().pop(name) for name in (\"Gone\",)} "; let built = match build_source( source, - "by_diff_metaconstant", + "by_diff_poppedclass", &toolchain, &dir, &Options { @@ -6843,30 +9105,37 @@ FIRST = Boundary.STRICT return; } }; - // nothing is declined: the class is emitted, and only the *construction* the runtime - // picks for it changes - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "Gone", + "`Gone` is rebound at module level, so installing this over it would replace what the rebind produced" + ), + ( + "Kept", + "`Kept` is rebound at module level, so installing this over it would replace what the rebind produced" + ) + ] + ); let out = run( &python, &dir, - "import by_diff_metaconstant as m\n\ - print(m.Boundary._member_names_)\n\ - print(isinstance(m.FIRST, m.Boundary), m.FIRST is m.Boundary.STRICT)\n\ - print(m.Boundary.STRICT.shout(), m.Boundary('strict') is m.Boundary.STRICT)\n\ - print([c.__name__ for c in m.Boundary.__mro__])\n\ - print(type(m.Boundary).__name__, type(m.Plain).__name__)\n\ - print(m.Plain._member_names_, m.Plain().shout() if False else 'plain')\n\ - print(type(m.Boundary.shout).__name__, type(m.Plain.shout).__name__)\n", + "import by_diff_poppedclass as m\n\ + print('Gone' in m.__dict__, 'Kept' in m.__dict__)\n\ + print(m.HIDDEN['Gone'].TAG, m.HIDDEN['Gone']().label(), m.Kept().label())\n\ + print(type(m.HIDDEN['Gone'].label).__name__, type(m.Kept.label).__name__)\n", ); assert_eq!( out, - "['STRICT', 'CONFORM']\n\ - True True\n\ - STRICT True\n\ - ['Boundary', 'StrEnum', 'str', 'ReprEnum', 'Enum', 'object']\n\ - EnumType EnumType\n\ - [] plain\n\ - function method_descriptor" + "False True\n\ + 1 gone kept\n\ + function function" ); } @@ -7030,9 +9299,12 @@ fn a_late_gift_that_could_hand_the_interpreted_class_back_is_left_alone() { // // so `SAMPLE` (an instance of the interpreted class), `ITEMS` (a list, which can be // given one after the question is asked) and `HIDDEN` (a tuple holding one) do not - // come across, while `MARKER`, `PAIR` and `shout` do. a dunder never does: a name in - // the type's dict does not fill a type slot, so `__ge__` there would answer - // `a.__ge__(b)` while `a >= b` still went to the slot. + // come across, while `MARKER`, `PAIR` and `shout` do. + // + // a dunder never comes across either — a name in the type's dict does not fill a type + // slot, so `__ge__` there would answer `a.__ge__(b)` while `a >= b` still went to the + // slot — and that is why a class the body hangs one on is turned down instead. `Ordered` + // is that half: dropping the entry is as wrong an answer as carrying it would be. // // `method_descriptor` is what says the compiled type answered at all: a class that // fell back to its interpreted definition would carry every one of these, because it @@ -7053,13 +9325,18 @@ class Held: return \"held\" +class Ordered: + def tag(self) -> str: + return \"ordered\" + + Held.MARKER = 3 Held.PAIR = Other Held.SAMPLE = Other() Held.ITEMS = [1, 2] Held.HIDDEN = (Other,) -Held.__ge__ = lambda self, right: True Held.shout = lambda self: self.tag().upper() +Ordered.__ge__ = lambda self, right: True "; let built = match build_source( source, @@ -7078,20 +9355,30 @@ Held.shout = lambda self: self.tag().upper() return; } }; - assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + assert_eq!( + built + .declined + .iter() + .map(|declined| declined.name.as_str()) + .collect::>(), + ["Ordered"] + ); let out = run( &python, &dir, "import by_diff_twinshapes as m\n\ print(m.Held.MARKER, m.Held.PAIR is m.Other, m.Held().shout())\n\ print(hasattr(m.Held, 'SAMPLE'), hasattr(m.Held, 'ITEMS'), hasattr(m.Held, 'HIDDEN'))\n\ - print(type(m.Held.tag).__name__, type(m.Other.tag).__name__)\n", + print(m.Ordered() >= m.Ordered(), m.Ordered().tag())\n\ + print(type(m.Held.tag).__name__, type(m.Other.tag).__name__,\n\ + \x20 type(m.Ordered.tag).__name__)\n", ); assert_eq!( out, "3 True HELD\n\ False False False\n\ - method_descriptor method_descriptor" + True ordered\n\ + method_descriptor method_descriptor function" ); } @@ -7403,17 +9690,19 @@ class Plain: } #[test] -fn a_subclass_of_a_class_the_metaclass_gates_turn_down_is_built_here() { - // both metaclass gates are asked while the layouts settle, so a class either of them - // turns down leaves the layout set — and its subclass is then laid out on the - // interpreted definition the way every other declining class's subclass is. asked - // while the body was lowered instead, the base stayed in the set and each subclass - // cascaded behind it, which is what this pair of declines used to be four of. +fn a_subclass_of_a_class_the_metaclass_gate_turns_down_is_built_here() { + // the metaclass gate is asked while the layouts settle, so a class it turns down + // leaves the layout set — and its subclass is then laid out on the interpreted + // definition the way every other declining class's subclass is. asked while the body + // was lowered instead, the base stayed in the set and the subclass cascaded behind + // it, which is what this decline used to be two of. // - // the bases here carry `ABCMeta`, so `PyType_FromSpecWithBases` is closed to the - // subclass and its metaclass builds it — `method_descriptor` is what says that - // construction happened at all, since a subclass that fell back would answer every - // value here from a `function` + // `Constant` is the other half, and it is the boundary: a class-level constant no + // longer turns a class down, so both it and its subclass are built here. the bases + // carry `ABCMeta`, so `PyType_FromSpecWithBases` is closed to every class in this + // module and the metaclass builds them — `method_descriptor` is what says that + // construction happened at all, since a class that fell back would answer from a + // `function` let Some((python, toolchain)) = environment() else { return; }; @@ -7462,7 +9751,7 @@ class BelowConstant(Constant): return; } }; - // exactly the two bases, and nothing behind them: a subclass in this list is the + // exactly the one base, and nothing behind it: a subclass in this list is the // cascade this move exists to stop assert_eq!( built @@ -7470,7 +9759,7 @@ class BelowConstant(Constant): .iter() .map(|declined| declined.name.as_str()) .collect::>(), - ["Decorated", "Constant"] + ["Decorated"] ); let out = run( &python, @@ -7480,7 +9769,8 @@ class BelowConstant(Constant): print(m.BelowConstant().size(), m.BelowConstant.TAG, m.BelowConstant().label())\n\ print([b.__name__ for b in m.BelowConstant.__mro__])\n\ print(isinstance(m.BelowConstant(), m.Constant), isinstance(m.BelowDecorated(), m.Decorated))\n\ - print(type(m.BelowDecorated.size).__name__, type(m.BelowConstant.size).__name__)\n", + print(type(m.BelowDecorated.size).__name__, type(m.BelowConstant.size).__name__,\n\ + \x20 type(m.Constant.label).__name__)\n", ); assert_eq!( out, @@ -7488,22 +9778,22 @@ class BelowConstant(Constant): 2 1 constant\n\ ['BelowConstant', 'Constant', 'object']\n\ True True\n\ - method_descriptor method_descriptor" + method_descriptor method_descriptor method_descriptor" ); } #[test] -fn a_pair_the_body_cross_links_agrees_when_a_gate_took_their_base_out_of_the_layouts() { - // `urllib.parse`'s shape, and the one that reverted this move the first time: the - // base declines at the class-level-constant gate, so both result classes are built - // here — over an interpreted base whose metaclass is `type`, which means a real - // emitted type replaces each twin in the namespace. `_pair` then runs against the - // twins, because the whole module body runs before module init installs anything. +fn a_pair_the_body_cross_links_agrees_when_the_link_is_made_after_the_class_statement() { + // `urllib.parse`'s shape, and the one that reverted this move the first time: `_pair` + // runs against the twins, because the whole module body runs before module init + // installs anything, and it hangs each result class off the other under a name no + // class body wrote. // // what makes it agree is that an emitted type carries what the body gave its twin // *and* remaps a carried twin to the type standing in for it — without the remap // `Text._encoded_counterpart()` builds something `isinstance` says is not a - // `m.Bytes` + // `m.Bytes`. the base used to decline at the class-level-constant gate and take the + // whole chain with it; it is emitted now, which is what the empty decline list says let Some((python, toolchain)) = environment() else { return; }; @@ -7563,15 +9853,15 @@ _pair() return; } }; - // `Root` and `Extra` follow `Mixin` down because an emitted class cannot have an - // interpreted subclass. `Text` and `Bytes` are the two that must not be here + // `Mixin` used to decline and take `Root` and `Extra` down with it, because an + // emitted class cannot have an interpreted subclass. none of the five is here now assert_eq!( built .declined .iter() .map(|declined| declined.name.as_str()) .collect::>(), - ["Mixin", "Root", "Extra"] + Vec::<&str>::new() ); let out = run( &python, @@ -7594,41 +9884,175 @@ _pair() } #[test] -fn a_private_name_in_a_class_body_is_mangled() { - // python binds an identifier of two leading underscores written in `class C` as - // `_C__spam`, whatever it names. the compiler read the written name, so the compiled - // class published `__read` and `__buffer` where python publishes `_Printer__read` - // and `_Printer__buffer` — `symtable.Function` lost five class attributes and three - // methods that way, and every read of one inside a method raised `AttributeError` - // because the name it looked for was never bound +fn a_private_name_in_a_class_body_is_mangled() { + // python binds an identifier of two leading underscores written in `class C` as + // `_C__spam`, whatever it names. the compiler read the written name, so the compiled + // class published `__read` and `__buffer` where python publishes `_Printer__read` + // and `_Printer__buffer` — `symtable.Function` lost five class attributes and three + // methods that way, and every read of one inside a method raised `AttributeError` + // because the name it looked for was never bound + // + // `_Printer` also exercises the class's *own* leading underscore, which the mangling + // strips + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_privatemangle"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class _Printer: + __LIMIT = 4 + + def __init__(self, source: str) -> None: + self.__buffer = source + self.plain = 0 + + def __read(self) -> str: + return self.__buffer + + def take(self) -> str: + return self.__read() + + def limit(self) -> int: + return self.__LIMIT +"; + let built = match build_source( + source, + "by_diff_privatemangle", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_privatemangle as m\n\ + p = m._Printer('hi')\n\ + print(p.take(), p.limit(), getattr(p, '_Printer__buffer'))\n\ + print(hasattr(p, '__buffer'), hasattr(m._Printer, '__read'))\n\ + print(m._Printer._Printer__LIMIT, callable(m._Printer._Printer__read))\n\ + print(type(m._Printer.take).__name__)\n", + ); + assert_eq!( + out, + "hi 4 hi\n\ + False False\n\ + 4 True\n\ + method_descriptor" + ); +} + +#[test] +fn a_name_that_is_both_a_class_constant_and_a_field_keeps_the_interpreted_class() { + // the constant is copied into the type's dict *after* `PyType_Ready` put the field's + // descriptor there, so the constant wins and every instance answers the class-level + // value instead of its own. that is a silent wrong answer, so the class declines and + // the interpreted definition — which python's own rules already get right — answers + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_constantfieldclash"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Tagged: + KIND: str = \"class-level\" + + def __init__(self, kind: str) -> None: + self.KIND = kind + + def read(self) -> str: + return self.KIND +"; + let built = match build_source( + source, + "by_diff_constantfieldclash", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!( + built.declined.iter().any(|declined| declined + .reason + .contains("both a class-level constant and a field")), + "declined: {:?}", + built.declined + ); + let out = run( + &python, + &dir, + "import by_diff_constantfieldclash as m\n\ + print(m.Tagged('mine').KIND, m.Tagged('mine').read(), m.Tagged.KIND)\n\ + print(type(m.Tagged.read).__name__)\n", + ); + assert_eq!(out, "mine mine class-level\nfunction"); +} + +#[test] +fn a_decorated_class_carries_the_body_its_own_decorator_was_handed() { + // a class-level constant is copied off the body the interpreted `class` statement + // wrote, taken while that statement runs and before any of the class's decorators is + // handed it. reading the finished definition instead is what this used to do, and by + // then the decorator has been over it. // - // `_Printer` also exercises the class's *own* leading underscore, which the mangling - // strips + // `@dataclass` shows both halves of what that cost. `later` has no default, so + // `_process_class` *deletes* the `field(init=False)` and left nothing to copy — the + // emitted type's annotation then read as a required argument after a defaulted one, + // and the module raised `TypeError` at import. `hidden` has one, so the `Field` was + // replaced by the bare `2` and `repr=False` went with it — the emitted `repr` showed + // a field the interpreted one hides, which no sweep can see. + // + // the descriptors are what say the compiled types answered: an interpreted leg has a + // plain `function` in both places let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_privatemangle"); + let dir = std::env::temp_dir().join("by_diff_decoratedconstant"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class _Printer: - __LIMIT = 4 +from dataclasses import dataclass, field - def __init__(self, source: str) -> None: - self.__buffer = source - self.plain = 0 - def __read(self) -> str: - return self.__buffer +@dataclass +class Sub: + tag: str = \"b\" + later: list[str] = field(init=False) - def take(self) -> str: - return self.__read() + def name(self) -> str: + return self.tag - def limit(self) -> int: - return self.__LIMIT + +@dataclass +class Holder: + shown: int = 1 + hidden: int = field(default=2, repr=False) + + def total(self) -> int: + return self.shown + self.hidden "; let built = match build_source( source, - "by_diff_privatemangle", + "by_diff_decoratedconstant", &toolchain, &dir, &Options { @@ -7647,46 +10071,63 @@ class _Printer: let out = run( &python, &dir, - "import by_diff_privatemangle as m\n\ - p = m._Printer('hi')\n\ - print(p.take(), p.limit(), getattr(p, '_Printer__buffer'))\n\ - print(hasattr(p, '__buffer'), hasattr(m._Printer, '__read'))\n\ - print(m._Printer._Printer__LIMIT, callable(m._Printer._Printer__read))\n\ - print(type(m._Printer.take).__name__)\n", + "from dataclasses import fields\n\ + import by_diff_decoratedconstant as m\n\ + print(m.Sub('x').name(), repr(m.Holder()), m.Holder().total())\n\ + print([(f.name, f.init, f.repr) for f in fields(m.Sub)])\n\ + print([(f.name, f.init, f.repr) for f in fields(m.Holder)])\n\ + print(type(m.Sub.name).__name__, type(m.Holder.total).__name__)\n", ); assert_eq!( out, - "hi 4 hi\n\ - False False\n\ - 4 True\n\ - method_descriptor" + "x Holder(shown=1) 3\n\ + [('tag', True, True), ('later', False, True)]\n\ + [('shown', True, True), ('hidden', True, False)]\n\ + method_descriptor method_descriptor" ); } #[test] -fn a_name_that_is_both_a_class_constant_and_a_field_keeps_the_interpreted_class() { - // the constant is copied into the type's dict *after* `PyType_Ready` put the field's - // descriptor there, so the constant wins and every instance answers the class-level - // value instead of its own. that is a silent wrong answer, so the class declines and - // the interpreted definition — which python's own rules already get right — answers +fn the_class_body_capture_reaches_only_this_module_s_own_body() { + // the capture is a copy of the builtins mapping, carrying a `__build_class__` of ours, + // put in this module's dict for the length of the fallback run — so no other module and + // no other thread can reach it, which on a free-threaded interpreter is the difference + // between a scoped trick and a race. + // + // python gives a function the builtins its defining frame had, though, so every + // function the body defined holds that copy for as long as it lives. `make_held` is + // that function: calling it once the import is over must make an ordinary class, with + // nothing recorded into a mapping that has been released by then. + // + // it is also a class named `Held`, the same as the module-level one, and the body calls + // it before the module is finished. that inner one must not stand in for the outer: + // `EARLY` is the interpreted answer taken during the body, `Held.kind` is what the + // emitted type carried, and they say different things let Some((python, toolchain)) = environment() else { return; }; - let dir = std::env::temp_dir().join("by_diff_constantfieldclash"); + let dir = std::env::temp_dir().join("by_diff_capturescope"); let _ = std::fs::remove_dir_all(&dir); let source = "\ -class Tagged: - KIND: str = \"class-level\" +class Held: + kind = \"module\" - def __init__(self, kind: str) -> None: - self.KIND = kind + def where(self) -> str: + return \"outer\" - def read(self) -> str: - return self.KIND + +def make_held(): + class Held: + kind = \"local\" + + return Held + + +EARLY = make_held().kind "; let built = match build_source( source, - "by_diff_constantfieldclash", + "by_diff_capturescope", &toolchain, &dir, &Options { @@ -7701,21 +10142,27 @@ class Tagged: return; } }; + // `make_held` writes a class in a function body, which nothing lowers. the `Held` the + // module itself writes is the one that has to be emitted assert!( - built.declined.iter().any(|declined| declined - .reason - .contains("both a class-level constant and a field")), + !built + .declined + .iter() + .any(|declined| declined.name == "Held"), "declined: {:?}", built.declined ); let out = run( &python, &dir, - "import by_diff_constantfieldclash as m\n\ - print(m.Tagged('mine').KIND, m.Tagged('mine').read(), m.Tagged.KIND)\n\ - print(type(m.Tagged.read).__name__)\n", + "import builtins\n\ + import by_diff_capturescope as m\n\ + print(m.Held.kind, m.EARLY, m.make_held().kind)\n\ + print(type(m.Held.where).__name__)\n\ + print(m.__builtins__ is builtins.__dict__)\n\ + print(m.make_held.__builtins__['__build_class__'] is builtins.__build_class__)\n", ); - assert_eq!(out, "mine mine class-level\nfunction"); + assert_eq!(out, "module local local\nmethod_descriptor\nTrue\nTrue"); } #[test] @@ -8023,7 +10470,12 @@ fn a_class_whose_base_declined_declines_with_it() { // a class was emitted with its base *silently dropped* when that base declined: // codegen looked the base up among the emitted classes, found nothing, and built a // type with no bases at all. the subclass then lost everything the base brought, - // which is a wrong answer rather than a slow one + // which is a wrong answer rather than a slow one. + // + // `Inner` keeps its base by naming it: it stores nothing of its own, so it is built + // on whatever the name holds at import — the interpreted `Outer` — exactly as a + // class over a base out of this module is. `method_descriptor` against `function` + // is what says which of the two answered let Some((python, toolchain)) = environment() else { return; }; @@ -8067,19 +10519,23 @@ class Inner(Outer): declined.sort_unstable(); assert_eq!( declined, - vec![ - ("Inner", "`Outer` declined, so it is not a base to build on"), - ("Outer", "`__new__` fills a type slot with no adapter yet"), - ] + vec![("Outer", "`__new__` fills a type slot with no adapter yet")] ); let out = run( &python, &dir, "import by_diff_declinedbase as m\n\ print(issubclass(m.Inner, m.Outer), isinstance(m.Inner(), m.Outer))\n\ - print(m.Inner().label(), m.Inner().tag())\n", + print(m.Inner().label(), m.Inner().tag())\n\ + print(type(m.Inner.__dict__['tag']).__name__,\n\ + \x20 type(m.Outer.__dict__['label']).__name__)\n", + ); + assert_eq!( + out, + "True True\n\ + outer inner\n\ + method_descriptor function" ); - assert_eq!(out, "True True\nouter inner"); } #[test] @@ -8482,6 +10938,9 @@ class Listc(A): assert_eq!( declined, vec![ + // `A` is not here on its own account any more: its `@classmethod` carries + // its convention on the method table entry, and it reads no slot zero. it + // is at the end of the list instead, behind the subclasses that declined ( "Cm", "`classmethod` and `staticmethod` both leave something other than the receiver in slot zero" @@ -8522,9 +10981,9 @@ class Listc(A): "Listc", "a `super()` in a comprehension reads that comprehension's own frame, which only python 3.12 and later fold into the method's" ), - // `A` is the base every class above extends, and `Cm` is the first of them - // to decline — so the base goes interpreted with it, or `Cm`'s interpreted - // definition would be subclassing a compiled type that refuses to be a base + // every one of those is a class the fallback leaves interpreted, and each + // extends `A` — so an emitted `A` would be a base with interpreted + // subclasses, which a static type refuses to be ( "A", "`Cm` declined, so it extends the interpreted definition rather than this type" @@ -8533,11 +10992,7 @@ class Listc(A): ); // a declined class still answers, through the interpreted definition the fallback // left behind. only the two whose answer does not turn on the interpreter version - // are called: python raises in most of the rest, and differently across versions. - // - // `Cm->Acgo` is also what says the decline was needed — a compiled `classmethod` - // is a method descriptor behind one, which raises `TypeError` when a *type* is - // what reaches it + // are called: python raises in most of the rest, and differently across versions let out = run( &python, &dir, @@ -10115,6 +12570,90 @@ def free(x: int, step: int = 1) -> int: ); } +#[test] +fn an_arity_error_is_worded_by_the_interpreter() { + // python's arity wording has rules a reimplementation keeps getting one short of: + // `and` from two names up, a comma *before* that `and` from three up, a range + // rather than a count once a parameter has a default, and a near miss offered to a + // caller who spelled a keyword almost right. the runtime does not reproduce them — + // it builds a function of the same shape and lets the interpreter refuse the call + // + // the three-name cases are what prove that happened. the wording the runtime keeps + // for itself is deliberately one comma short of python's, so a leg that answered + // from it rather than from the shape would differ here rather than pass + agree_python( + "arityword", + "\ +class Bare: + pass + + +class Three: + def __init__(self, a: int, b: int, c: int) -> None: + self.a = a + + +class Spread: + def __init__(self, a: int, b: int = 1, c: int = 2) -> None: + self.a = a + + +class Box: + def __init__(self) -> None: + self.n = 0 + + def take(self, x: int, y: int, z: int) -> int: + return x + y + z + + +def one(a: int) -> int: + return a + + +def two(a: int, b: int) -> int: + return a + + +def four(a: int, b: int, c: int, d: int) -> int: + return a + + +def named(a: int, *, b: int, c: int, d: int) -> int: + return a + + +def rest(a: int, b: int, *more: int) -> int: + return a +", + &[ + // one name, then two, then three: the separator changes at each step + "str(_capture(m.one))", + "str(_capture(m.two))", + "str(_capture(m.four))", + "str(_capture(m.named, 1))", + "str(_capture(m.rest))", + // a receiver is counted in the arity sentence and not in this one + "str(_capture(lambda: m.Box().take()))", + "str(_capture(lambda: m.Box().take(1, 2, 3, 4)))", + "str(_capture(m.Three))", + "str(_capture(lambda: m.Three(1)))", + // a default turns the count into a range + "str(_capture(lambda: m.Spread(1, 2, 3, 4)))", + // `object.__init__` refuses anything at all, and does not say which kind + "str(_capture(lambda: m.Bare(1)))", + "str(_capture_kw(m.Bare, (), {'q': 1}))", + // a keyword that nearly names a parameter is offered the parameter + "str(_capture_kw(m.two, (1, 2), {'aa': 1}))", + "str(_capture_kw(m.Three, (1, 2, 3), {'aa': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(1, 2, 3, **k), (), {'xx': 1}))", + // and one that nearly names the *synthetic* receiver must not be offered it + "str(_capture_kw(m.two, (1, 2), {'_by_self': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(1, 2, 3, **k), (), {'_by_rest': 1}))", + "str(_capture_kw(lambda **k: m.Box().take(**k), (), {'_by_self': 1}))", + ], + ); +} + #[test] fn a_with_inside_a_generator_agrees() { // the manager has to survive every suspension the body makes, so it lives in a @@ -10390,14 +12929,17 @@ fn a_nested_function_may_be_decorated_or_generic() { return; } // a decorator on a nested function wraps the closure where the `def` stands, - // outermost last — the same order the statement itself applies them. the name - // is resolved the way the module-level ones are, as `LOAD_GLOBAL` would. + // outermost last — the same order the statement itself applies them. the whole + // *expression* is evaluated there, in this frame, which is what lets a call, a + // dotted name and a name the frame itself binds all be taken: `@wraps(fn)` reads + // `fn` out of a register, and `@held` reads the parameter holding the decorator. // // a type parameter is erased here as anywhere else, so a generic nested // function needed nothing beyond dropping the decline agree_python( "nesteddec", "\ +import functools from typing import Callable @@ -10446,6 +12988,38 @@ def counted(n: int) -> list[int]: out.append(each(0)) return out + + +def by_call(n: int) -> Callable[[int], int]: + @scaled(3) + def inner(x: int) -> int: + return x + n + + return inner + + +def by_dotted(n: int) -> Callable[[int], int]: + @functools.cache + def inner(x: int) -> int: + return x + n + + return inner + + +def by_held(n: int, held: Callable[[Callable[[int], int]], Callable[[int], int]]) -> Callable[[int], int]: + @held + def inner(x: int) -> int: + return x + n + + return inner + + +def scaled(k: int) -> Callable[[Callable[[int], int]], Callable[[int], int]]: + def outer(fn: Callable[[int], int]) -> Callable[[int], int]: + def wrapper(x: int) -> int: + return fn(x) * k + return wrapper + return outer ", &[ "m.offset(10)(1)", @@ -10457,6 +13031,13 @@ def counted(n: int) -> list[int]: // each iteration decorates its own closure "m.counted(4)", "m.counted(0)", + // a decorator that is a call, evaluated where the `def` stands + "m.by_call(10)(1)", + "m.by_call(0)(5)", + // a dotted name, read off a module this one imported + "m.by_dotted(10)(1)", + // and one the frame is *holding*, which a global lookup would have missed + "m.by_held(10, m.twice)(1)", ], ); } @@ -11091,7 +13672,7 @@ fn the_arithmetic_dunders_reach_both_directions() { // whether this is `__add__` or `__radd__`. one direction the class does not // define answers `NotImplemented` agree_python( - "arith", + "arithdunders", "\ class Vec: def __init__(self, x: int, y: int) -> None: @@ -11310,7 +13891,7 @@ fn the_container_dunders_fill_their_slots() { // `__setitem__` share the mapping sub-table with `__len__`, and `__contains__` // needs a sequence table of its own agree_python( - "containers", + "containerdunders", "\ class Grid: def __init__(self, items: list[int]) -> None: @@ -12942,30 +15523,226 @@ data class Plain: def doubled(self) -> int: return self.n * 2 -def use(n: int) -> str: - p = Point(n, n + 1) - return str(p.total()) + str(Point.tag) +def use(n: int) -> str: + p = Point(n, n + 1) + return str(p.total()) + str(Point.tag) + +def pairs(s: str) -> str: + q = Pair(s, s) + return q.a + q.b + str(Pair.tag) + str(Pair.made) +", + &[ + "m.use(1)", + "m.pairs('z')", + "m.Point(1, 2).total()", + "m.Point(1, 2).x", + "m.Point.tag", + // both decorators ran, innermost first + "(m.Pair.tag, m.Pair.made)", + // an undecorated class is untouched and keeps its direct method call + "m.Plain(3).doubled()", + // a decorated one is mutable, which is the whole reason it is a heap + // type — an undecorated one stays static, and rejecting `setattr` there + // is a difference from the interpreted class that predates this + "[type(e).__name__ for e in [_capture(setattr, m.Point, 'extra', 1)]]", + "m.Point.extra if hasattr(m.Point, 'extra') else None", + ], + ); +} + +#[test] +fn a_class_decorator_is_handed_the_body_the_class_statement_wrote() { + // a decorator that only *reads* its class still reads the whole of it: the names the + // body bound, the values it gave them and the annotations it wrote. the annotations + // are the ones an emitted type gets last — they are carried over from the twin, and + // that carrying used to happen *after* every decorator had run, so each one was + // handed a class whose body had not arrived yet + agree_python( + "decoratorreads", + "\ +SEEN = {} + + +def inspecting(cls: type) -> type: + SEEN[cls.__name__] = (cls.__annotations__, 'helper' in cls.__dict__, cls.limit) + return cls + + +@inspecting +class Widget: + tag: str + size: int + limit = 7 + + def helper(self) -> int: + return 4 +", + &[ + "sorted(m.SEEN['Widget'][0].items(), key=str)", + "m.SEEN['Widget'][1]", + "m.SEEN['Widget'][2]", + "m.Widget.limit", + "m.Widget().helper()", + ], + ); +} + +#[test] +fn a_dataclass_decorator_builds_a_constructor_that_runs() { + // `@dataclass` does not read a class so much as *generate* from it — an `__init__` + // taking one argument per annotation and assigning one attribute each — so it is the + // strictest question that can be asked about how faithful an emitted class is. it + // found two answers wrong at once: an empty `__annotations__` made it write + // `__init__(self)`, and once that was fixed the assignments had nowhere to land, + // because an emitted instance is its layout and this class has no fields at all. + // + // `y` carries a default, which is a class-level value the decorator reads and then + // rewrites: `_process_class` leaves the bare `5` where the `field(default=5)` stood. + // so the value the emitted type carries has to be the one the body wrote rather than + // the one left behind, and `_capture(m.Point)` is where that shows — a `Point` built + // with no arguments at all + agree_python( + "dataclassdecorator", + "\ +from dataclasses import dataclass, field, fields, replace + + +@dataclass +class Point: + x: int + y: int = field(default=5, repr=False) + + def total(self) -> int: + return self.x + self.y + + +def make(n: int) -> str: + return repr(Point(n, 5)) +", + &[ + "m.make(3)", + "m.Point(1, 2)", + "m.Point(1)", + "m.Point(1, 2) == m.Point(1, 2)", + "m.Point(1, 2) == m.Point(1, 3)", + "m.Point(1, 2).total()", + "[(f.name, f.type, f.repr) for f in m.fields(m.Point)]", + "sorted(m.Point(1, 2).__dict__.items())", + "m.Point.__match_args__", + "m.replace(m.Point(1, 2), y=9)", + "[(type(e).__name__, str(e)) for e in [_capture(m.Point)]]", + ], + ); +} + +#[test] +fn a_decorated_class_is_the_compiled_one_and_its_dict_is_collectable() { + // the differential legs agree whichever class answered, so this is where the + // compiled one is pinned down. it is not a formality: the managed dict this class + // now carries moves `tp_dictoffset` off its base's, which `By_OffsetsHoldUp` read as + // the wrong kind of inheritance — the type was quietly dropped for its interpreted + // twin, and every compiled function went on reading that twin's instances as its own + // struct. `m.on_final(m.Fixed(2, 'x'))` answered a pointer + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_decoratedlive"); + let _ = std::fs::remove_dir_all(&dir); + // `from __future__ import annotations` is what keeps `through`'s parameter from + // being a *read* of `Derived`: without it the module evaluates the annotation where + // the `def` stands, which is inside the window where `Derived` still holds the + // interpreted definition, and the class declines rather than move its decorator + let source = "\ +from __future__ import annotations + +from dataclasses import dataclass + + +def tagged(cls: type) -> type: + cls.tag = 'seen' + return cls + + +@dataclass +class Point: + x: int + + def doubled(self) -> int: + return self.x * 2 + + +@tagged +class Held: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +class Plain: + def __init__(self, n: int) -> None: + self.n = n + + def read(self) -> int: + return self.n + + +@tagged +class Derived(Plain): + def tripled(self) -> int: + return self.n * 3 -def pairs(s: str) -> str: - q = Pair(s, s) - return q.a + q.b + str(Pair.tag) + str(Pair.made) -", - &[ - "m.use(1)", - "m.pairs('z')", - "m.Point(1, 2).total()", - "m.Point(1, 2).x", - "m.Point.tag", - // both decorators ran, innermost first - "(m.Pair.tag, m.Pair.made)", - // an undecorated class is untouched and keeps its direct method call - "m.Plain(3).doubled()", - // a decorated one is mutable, which is the whole reason it is a heap - // type — an undecorated one stays static, and rejecting `setattr` there - // is a difference from the interpreted class that predates this - "[type(e).__name__ for e in [_capture(setattr, m.Point, 'extra', 1)]]", - "m.Point.extra if hasattr(m.Point, 'extra') else None", - ], + +def through(d: Derived) -> int: + return d.tripled() +"; + let built = match build_source( + source, + "by_diff_decoratedlive", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import gc\n\ + import by_diff_decoratedlive as m\n\ + print(type(m.Point.doubled).__name__, type(m.Held.read).__name__,\n\ + \x20 type(m.Plain.read).__name__, type(m.Derived.tripled).__name__)\n\ + print(m.Point(3).doubled(), m.Held(4).read(), m.Plain(5).read())\n\ + print(m.Point(3).__dict__, m.Held.tag)\n\ + print(m.through(m.Derived(2)), m.Derived(2).read())\n\ + gc.collect()\n\ + base = len(gc.get_objects())\n\ + for _ in range(200):\n\ + \x20 h = m.Held(1)\n\ + \x20 h.itself = h\n\ + del h\n\ + gc.collect()\n\ + print('collected' if len(gc.get_objects()) <= base + 10 else 'leaked')\n\ + print(gc.is_tracked(m.Held(1)), gc.is_tracked(m.Plain(1)))\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor method_descriptor method_descriptor\n\ + 6 4 5\n\ + {'x': 3} seen\n\ + 6 2\n\ + collected\n\ + True False" ); } @@ -14180,3 +16957,311 @@ def carried(n: int) -> object: ], ); } + +#[test] +fn a_module_level_name_bound_to_a_class_follows_the_class_it_named() { + // the whole module body runs against the interpreted definitions, so a name it binds + // to a class holds that definition — and the compiled type only ever replaces the one + // name the `class` statement wrote. what that left was two classes of the same name in + // the same module: `Kind()` built an object `isinstance(obj, C)` denied, and the + // compiled `hi` refused it outright. + // + // a name that *is* a twin is the one shape that moves soundly, so it is moved onto + // whatever stands under the class's own name. the annotation on it is beside the + // point: `Kind: type[C] = C` and `Bare = C` are the same binding. + // + // `method_descriptor` is what says the compiled type answered at all — a class that + // fell back to its interpreted definition has one class and could not show this + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_classalias"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class C: + def hi(self) -> int: + return 7 + + +Kind: type[C] = C +Bare = C +Rebound = Bare +"; + let built = match build_source( + source, + "by_diff_classalias", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_classalias as m\n\ + print(m.Kind is m.C, m.Bare is m.C, m.Rebound is m.C)\n\ + print(isinstance(m.Kind(), m.C), isinstance(m.C(), m.Bare))\n\ + print(m.C.hi(m.Kind()), m.Kind().hi())\n\ + print(type(m.C.hi).__name__)\n", + ); + assert_eq!( + out, + "True True True\n\ + True True\n\ + 7 7\n\ + method_descriptor" + ); +} + +#[test] +fn a_class_constant_naming_another_class_is_the_type_that_replaced_it() { + // a class-level constant is taken off the interpreted definition, so `attr = C` in a + // body hands over the *twin* — and copying that verbatim gave the compiled type an + // attribute naming a class nothing else in the module could reach. it goes through the + // same substitution every carried attribute does. + // + // `held` is the boundary in the other direction: a constant that only *reaches* a twin + // cannot be substituted, because the tuple is the object the body built and its + // identity is not the twin's. it is pinned here as still *present*, because dropping + // those instead was built and backed out on the measurement — 65 attributes lost over + // the stdlib, `ipaddress`'s network constants among them. what the reach holds is a + // defect of its own and is left exactly as the interpreted definition had it + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_classconst"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class C: + def hi(self) -> int: + return 7 + + +class Holder: + attr = C + held = (C,) + plain = 3 + + def tag(self) -> str: + return \"holder\" +"; + let built = match build_source( + source, + "by_diff_classconst", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "declined: {:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_classconst as m\n\ + print(m.Holder.attr is m.C, m.Holder.plain)\n\ + print(m.C.hi(m.Holder.attr()))\n\ + print(hasattr(m.Holder, 'held'), len(m.Holder.held))\n\ + print(type(m.C.hi).__name__, type(m.Holder.tag).__name__)\n", + ); + assert_eq!( + out, + "True 3\n\ + 7\n\ + True 1\n\ + method_descriptor method_descriptor" + ); +} + +#[test] +fn a_base_written_as_an_alias_reaches_the_class_it_was_bound_to() { + // `Alias` and `Root` are one class, so the two spellings have to build one class. + // taking the alias for a name out of this module built `Over` on the interpreted + // definition instead — the emitted type goes into the namespace under `Root`, and an + // alias is carried over to it only once every class has been built — so + // `isinstance(Over(), Root)` answered `False` where python answers `True`, while + // `Over.__mro__` still said `Root`. a wrong answer, and one no sweep reaches: the + // class builds, imports, constructs and subclasses with only its contents lying + agree_python( + "aliasbase", + "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +", + &[ + "isinstance(m.Over(), m.Root)", + "m.Over.__mro__[1] is m.Root", + "[c.__name__ for c in m.Over.__mro__]", + "m.Over().root()", + "m.Over().side()", + "issubclass(m.Over, m.Root)", + ], + ); +} + +#[test] +fn an_alias_reaches_the_compiled_type_rather_than_the_twin() { + // the same source again, because `agree` cannot say which build answered: the + // interpreted definition answers every one of those calls identically. what is + // wanted is that the *compiled* type stands under the name, and `method_descriptor` + // against `function` is what says so + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_aliasbase_type"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + let built = match build_source( + source, + "by_diff_aliasbase_type", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + assert!(built.declined.is_empty(), "{:?}", built.declined); + let out = run( + &python, + &dir, + "import by_diff_aliasbase_type as m\n\ + print(type(m.Root.root).__name__, type(m.Over.side).__name__)\n\ + print(m.Alias is m.Root, m.Over.__mro__[1] is m.Root)\n\ + print(isinstance(m.Over(), m.Root))\n", + ); + assert_eq!( + out, + "method_descriptor method_descriptor\n\ + True True\n\ + True" + ); +} + +#[test] +fn an_alias_does_not_carry_a_base_this_module_lays_out_past_the_gate() { + // `class OnLaid(Laid, codecs.Codec)` is refused because the layout would have to be + // inherited from outside and laid out here at once. written through an alias it went + // straight past — the gate asks `layouts` about the *name* — and compiled the one + // shape it exists to refuse. so the refusal has to survive the spelling, and `Laid` + // has to go interpreted with it or `isinstance` disagrees for the same reason again + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = std::env::temp_dir().join("by_diff_aliaslaid"); + let _ = std::fs::remove_dir_all(&dir); + let source = "\ +import codecs + + +class Laid: + def __init__(self, n: int) -> None: + self.n = n + + def held(self) -> int: + return self.n + + +Alias = Laid + + +class OnLaid(Alias, codecs.Codec): + def side(self) -> int: + return 2 +"; + let built = match build_source( + source, + "by_diff_aliaslaid", + &toolchain, + &dir, + &Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + ) { + Ok(built) => built, + Err(error) => { + assert!(missing_toolchain(&error), "failed to build: {error:#}"); + eprintln!("skipping: no working C toolchain ({error})"); + return; + } + }; + let declined: Vec<(&str, &str)> = built + .declined + .iter() + .map(|declined| (declined.name.as_str(), declined.reason.as_str())) + .collect(); + assert_eq!( + declined, + vec![ + ( + "OnLaid", + "a base this module lays out cannot stand beside one it does not" + ), + ( + "Laid", + "`OnLaid` declined, so it extends the interpreted definition rather than this type" + ) + ] + ); + let out = run( + &python, + &dir, + "import by_diff_aliaslaid as m\n\ + print(m.Alias is m.Laid, m.OnLaid.__mro__[1] is m.Laid)\n\ + print(isinstance(m.OnLaid(1), m.Laid), m.OnLaid(1).held(), m.OnLaid(1).side())\n\ + print(type(m.Laid.held).__name__, type(m.OnLaid.side).__name__)\n", + ); + assert_eq!( + out, + "True True\n\ + True 1 2\n\ + function function" + ); +} diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 48b7056309..18ccc3e96a 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -18,7 +18,7 @@ use std::process::Command; use by_build::{Options, Toolchain, build_module, build_source}; use by_ir::builder::FunctionBuilder; -use by_ir::function::{CallConvention, ModuleIr}; +use by_ir::function::{CallConvention, ModuleIr, ModuleName}; use by_ir::ops::{BinOp, CmpOp, Op, Terminator, Value}; use by_ir::rtype::RType; @@ -114,7 +114,7 @@ fn arith_module() -> ModuleIr { builder.terminate(Terminator::Return(Value::Register(result))); ModuleIr { - name: "by_e2e_arith".to_string(), + name: by_ir::ModuleName::new("by_e2e_arith"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -178,7 +178,7 @@ fn fib_module() -> ModuleIr { builder.terminate(Terminator::Return(Value::Register(a))); ModuleIr { - name: "by_e2e_fib".to_string(), + name: by_ir::ModuleName::new("by_e2e_fib"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -314,7 +314,7 @@ fn division_floors_like_python_and_raises_on_zero() { }); builder.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "by_e2e_div".to_string(), + name: by_ir::ModuleName::new("by_e2e_div"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -370,7 +370,7 @@ fn floats_are_unboxed_and_exclude_int() { }); builder.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "by_e2e_float".to_string(), + name: by_ir::ModuleName::new("by_e2e_float"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -441,7 +441,7 @@ fn calls_between_compiled_functions_stay_native() { quad.terminate(Terminator::Return(Value::Register(twice))); let module = ModuleIr { - name: "by_e2e_call".to_string(), + name: by_ir::ModuleName::new("by_e2e_call"), functions: vec![double.finish(), quad.finish()], declined: Vec::new(), classes: Vec::new(), @@ -701,7 +701,7 @@ fn a_stdlib_module_compiles_without_a_hard_failure(name: &str) { }; match build_source( &source, - &module, + module.as_str(), &toolchain, &dir.join(&module), &Options::default(), @@ -1122,3 +1122,97 @@ fn the_emitted_c_names_no_pointer_type_it_does_not_mean() { "the C compiler rejected the generated code:\n{stderr}" ); } + +/// the source of a package member, whose answers say which member is speaking +/// +/// `tag` is the same name in both `dup` modules, which is the whole point: a flat +/// output directory called both files `dup`, so the second silently replaced the +/// first and neither was importable under the name it was compiled as +fn member_source(tag: i32) -> String { + format!( + "class Member:\n def __init__(self) -> None:\n self.tag: int = {tag}\n\n\ndef tag() -> int:\n return {tag}\n" + ) +} + +/// the artefacts of a package build have to be laid out as the package, because +/// that is the only shape cpython's finder will import them back under: it looks +/// for `pkg/sub/dup` for a member and `pkg/sub/__init__` for the +/// package itself, and never for a flat file named after the last component +#[test] +fn a_package_is_built_as_a_tree_and_imports_under_its_dotted_names() { + let Some((python, toolchain)) = environment() else { + return; + }; + if !supports(&toolchain, (3, 12)) { + return; + } + let dir = std::env::temp_dir().join("by_e2e_package_tree"); + let _ = std::fs::remove_dir_all(&dir); + + let options = Options { + language: by_irbuild::Language::Python, + ..Options::default() + }; + // both packages and both members go into one output directory, the way + // `by compile -o` builds a whole project + let members = [ + (ModuleName::package("by_e2e_pkg"), 1), + (ModuleName::package("by_e2e_pkg.sub"), 2), + (ModuleName::new("by_e2e_pkg.dup"), 3), + (ModuleName::new("by_e2e_pkg.sub.dup"), 4), + ]; + for (name, tag) in &members { + let Ok(built) = build_source( + &member_source(*tag), + name.clone(), + &toolchain, + &dir, + &options, + ) else { + eprintln!("skipping: no working C toolchain"); + return; + }; + assert!( + built.artifact.extension.exists(), + "{} was written to {}", + name.dotted(), + built.artifact.extension.display() + ); + } + + // four distinct artefacts: the two `dup` members used to be one file + assert_eq!( + members + .iter() + .map(|(name, _)| toolchain.extension_path(name)) + .collect::>() + .len(), + 4 + ); + + let printed = script( + &python, + &dir, + "import sys\n\ + import by_e2e_pkg.sub.dup\n\ + import by_e2e_pkg.dup\n\ + for name in ('by_e2e_pkg', 'by_e2e_pkg.sub', 'by_e2e_pkg.dup', 'by_e2e_pkg.sub.dup'):\n\ + \x20 m = sys.modules[name]\n\ + \x20 print(name, m.__name__, m.tag(), m.Member.__module__, m.__file__)\n", + ); + let lines: Vec<&str> = printed.lines().collect(); + assert_eq!(lines.len(), 4, "{printed}"); + for (line, (name, tag)) in lines.iter().zip(&members) { + let fields: Vec<&str> = line.split_whitespace().collect(); + // the key it answered to in `sys.modules`, the name the module reports, + // and the module a class written in it belongs to all have to be the one + // it was compiled as + assert_eq!(fields[0], name.dotted(), "{printed}"); + assert_eq!(fields[1], name.dotted(), "{printed}"); + assert_eq!(fields[2], tag.to_string(), "{printed}"); + assert_eq!(fields[3], name.dotted(), "{printed}"); + // and it answered from the extension, not from some interpreted source + // that happened to be lying beside it + assert!(fields[4].ends_with(&toolchain.ext_suffix), "{printed}"); + } +} diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index eb8ff0909e..8427a3a1de 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -26,7 +26,7 @@ use std::collections::BTreeSet; use std::fmt::Write; use by_ir::function::{ - ClassBase, ClassIr, Function, KeywordValue, ModuleIr, RegisterDecl, Surface, + Binding, ClassBase, ClassIr, Function, KeywordValue, ModuleIr, RegisterDecl, Surface, }; use by_ir::ops::{BinOp, BlockId, CmpOp, Mutation, Op, RegisterId, Terminator, UnaryOp, Value}; use by_ir::rtype::{Primitive, RType, tuple_mangle}; @@ -125,7 +125,7 @@ pub fn emit_module(module: &ModuleIr) -> String { let _ = writeln!( out, "static PyObject *{} = NULL;", - function.interpreted_symbol(&module.name) + function.interpreted_symbol(module.name.dotted()) ); } } @@ -181,7 +181,7 @@ pub fn emit_module(module: &ModuleIr) -> String { // so its name is a pointer rather than a static struct. only a generator's state // and a closure's environment stay static let heap = heap_type(module, class); - let type_name = class.type_name(&module.name); + let type_name = class.type_name(module.name.dotted()); let _ = writeln!( out, "{} {type_name};", @@ -218,7 +218,7 @@ pub fn emit_module(module: &ModuleIr) -> String { let _ = writeln!( out, "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames);", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } } @@ -233,8 +233,13 @@ pub fn emit_module(module: &ModuleIr) -> String { out.push('\n'); // a method's wrapper takes the receiver from the `self` slot rather // than from the argument vector, which is how `METH_FASTCALL` on a - // type presents it - out.push_str(&emit_wrapper(module, method, true)); + // type presents it. a `staticmethod` has nothing there and binds every + // parameter it declares, exactly as a module-level function does + out.push_str(&emit_wrapper( + module, + method, + method.binding.takes_slot_zero_from_self(), + )); out.push('\n'); } } @@ -332,6 +337,56 @@ fn emit_tuple_struct(module: &ModuleIr, items: &[RType]) -> String { out } +/// whether the collector can follow this field, which it can where the field is held as +/// a plain `PyObject *` +/// +/// a tagged `int` is either not an object at all or a `PyLong`, and neither can be part +/// of a cycle; an unboxed buffer holds no references either +fn collectable(field: &by_ir::function::FieldDecl) -> bool { + matches!( + field.ty, + RType::Instance { .. } + | RType::Primitive(Primitive::Object | Primitive::Str | Primitive::List) + ) +} + +/// `tp_traverse` and `tp_clear` for a class that owns its layout and keeps an instance +/// dict beside it +/// +/// the fields are this class's whole struct — a subclass's begins with its base's and is +/// cloned into it, so there is no base to chain to and nothing of the base's is missed. +/// the type is visited because an instance of a heap type holds a reference to it, and +/// the dict is visited because that is the whole reason the type is collected at all +fn emit_collected_instance(module: &ModuleIr, class: &ClassIr) -> String { + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); + let mut visits = String::new(); + let mut clears = String::new(); + for field in class.fields.iter().filter(|field| collectable(field)) { + let _ = writeln!( + out_slot(&mut visits), + " Py_VISIT(self->{});", + field.member() + ); + let _ = writeln!( + out_slot(&mut clears), + " Py_CLEAR(self->{});", + field.member() + ); + } + format!( + "static int {type_name}_traverse({struct_name} *self, visitproc visit, void *arg) {{\n\ + \x20 Py_VISIT(Py_TYPE(self));\n\ + {visits}\ + \x20 By_VisitManagedDict((PyObject *)self, visit, arg);\n\ + \x20 return 0;\n}}\n\n\ + static int {type_name}_clear({struct_name} *self) {{\n\ + {clears}\ + \x20 By_ClearManagedDict((PyObject *)self);\n\ + \x20 return 0;\n}}\n\n" + ) +} + /// `tp_dealloc`, `tp_traverse` and `tp_clear` for a class whose fields sit past a /// base's instance /// @@ -341,8 +396,8 @@ fn emit_tuple_struct(module: &ModuleIr, items: &[RType]) -> String { /// optional either: a base like `Exception` is a GC type, so ours is, and a field the /// collector cannot see holds its cycle alive forever fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); // the *declaring* type, not `Py_TYPE(self)`: a python subclass of this class is a // different type whose data area is somewhere else again, and the base to chain to // is this class's base rather than that subclass's @@ -350,19 +405,9 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { let fields = format!("({struct_name} *)By_TypeData(self, {declared})"); let mut out = String::new(); - // only a field the collector can follow, which is one held as a plain `PyObject *`. - // a tagged `int` is either not an object at all or a `PyLong`, and neither can be - // part of a cycle; an unboxed buffer holds no references either - let collected = |field: &&by_ir::function::FieldDecl| { - matches!( - field.ty, - RType::Instance { .. } - | RType::Primitive(Primitive::Object | Primitive::Str | Primitive::List) - ) - }; let mut visits = String::new(); let mut clears = String::new(); - for field in class.fields.iter().filter(collected) { + for field in class.fields.iter().filter(|field| collectable(field)) { let _ = writeln!( out_slot(&mut visits), " Py_VISIT(by_f->{});", @@ -375,9 +420,13 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { ); } - // `subtype_traverse` visits the type only when the base is not itself a heap type, - // which is exactly the case a subclass of this class is *not* in — so this one has - // to, or the type is never collected + // an instance of a heap type counts as a reference to that type, and the collector + // has to see it or a cycle through the type is never broken. exactly one traverse in + // the chain reports it, which is `subtype_traverse`'s own rule: the one whose base + // does not itself carry the link. a base out of this module is not a heap type — the + // construction refuses one that is — so this traverse is that one; a base *this* + // module appends to is, and its traverse has already reported it. counting it twice + // would tell the collector the instance holds two references where it holds one let _ = write!( out, "static int {type_name}_traverse(PyObject *self, visitproc visit, void *arg) {{\n\ @@ -388,7 +437,7 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { \x20 int by_r = by_base->tp_traverse(self, visit, arg);\n\ \x20 if (by_r) return by_r;\n\ \x20 }}\n\ - \x20 Py_VISIT(Py_TYPE(self));\n\ + \x20 if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_VISIT(Py_TYPE(self));\n\ \x20 return 0;\n}}\n\n\ static int {type_name}_clear(PyObject *self) {{\n\ \x20 {struct_name} *by_f = {fields};\n\ @@ -410,16 +459,21 @@ fn emit_appended_storage(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out_slot(&mut releases), " {release}"); } } + // and dropped by exactly one rung, the same one the traverse reports it from: a base + // that is itself a heap type has a deallocator of its own that drops it, and two + // drops for the one reference free the type underneath everything still using it let _ = write!( out, "static void {type_name}_dealloc(PyObject *self) {{\n\ \x20 PyTypeObject *by_type = Py_TYPE(self);\n\ + \x20 PyTypeObject *by_base = {declared}->tp_base;\n\ \x20 if (PyType_HasFeature(by_type, Py_TPFLAGS_HAVE_GC)) PyObject_GC_UnTrack(self);\n\ \x20 {{ {struct_name} *by_f = {fields};\n\ {releases}\ \x20 }}\n\ - \x20 {declared}->tp_base->tp_dealloc(self);\n\ - \x20 if (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE) Py_DECREF(by_type);\n}}\n\n" + \x20 by_base->tp_dealloc(self);\n\ + \x20 if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)\n\ + \x20 && (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_DECREF(by_type);\n}}\n\n" ); out } @@ -436,7 +490,7 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { }; let mut out = format!( "typedef struct {} {{\n{header}", - class.struct_name(&module.name) + class.struct_name(module.name.dotted()) ); for field in &class.fields { let _ = writeln!(out, " {} {};", ctype(module, &field.ty), field.member()); @@ -446,7 +500,7 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!(out, " char {};", field.presence()); } } - let _ = writeln!(out, "}} {};", class.struct_name(&module.name)); + let _ = writeln!(out, "}} {};", class.struct_name(module.name.dotted())); out } @@ -454,18 +508,27 @@ fn emit_class_struct(module: &ModuleIr, class: &ClassIr) -> String { /// releases them. every field is written by `__init__`, which is what makes them /// *always defined* — no bitfield and no per-read check fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); let mut out = String::new(); + let keeps_a_dict = instance_dict(module, class); if external_storage(module, class) { out.push_str(&emit_appended_storage(module, class)); } else { + if keeps_a_dict { + out.push_str(&emit_collected_instance(module, class)); + } // dealloc releases each refcounted field, then the object let _ = writeln!( out, "static void {type_name}_dealloc({struct_name} *self) {{" ); + // a collected instance is on the collector's list until it says otherwise, and + // a list holding a half-freed object is what the next collection walks + if keeps_a_dict { + out.push_str(" PyObject_GC_UnTrack(self);\n"); + } // a finalizer does not run itself: `subtype_dealloc` calls it, and a type that // writes its own dealloc has to do the same or the cleanups never happen. a // negative answer means the finalizer resurrected the object, and freeing it @@ -479,6 +542,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { " if (PyObject_CallFinalizerFromDealloc((PyObject *)self) < 0) return;\n", ); } + if keeps_a_dict { + out.push_str(" By_ClearManagedDict((PyObject *)self);\n"); + } for field in &class.fields { if let Some(release) = dec_ref(&field.ty, &format!("self->{}", field.member())) { let _ = writeln!(out, " {release}"); @@ -712,7 +778,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { out, " if ({}) return By_InitInterpreted({}, {fname}, selfobj, args, kwds);", tests.join(" || "), - init.interpreted_symbol(&module.name) + init.interpreted_symbol(module.name.dotted()) ); } let params = init.params().get(1..).unwrap_or_default(); @@ -779,7 +845,7 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { out, " {{ {} by_result = {}(({receiver})selfobj{arguments});", ctype(module, &init.ret), - init.native_symbol(&module.name) + init.native_symbol(module.name.dotted()) ); if init.convention.can_fail() { let _ = writeln!( @@ -850,8 +916,8 @@ fn mark_present(field: &by_ir::function::FieldDecl) -> String { /// the getters, setters, slot table and type spec python sees fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { - let struct_name = class.struct_name(&module.name); - let type_name = class.type_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); + let type_name = class.type_name(module.name.dotted()); let mut out = String::new(); // getters and setters, so python sees ordinary attributes @@ -921,6 +987,16 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { field.name, field.name ); } + // a class keeping an instance dict answers `__dict__` with it — but only where the + // dict is the whole of an instance's state. a class with fields of its own keeps + // them in its layout, and a mapping that named none of them would be an *empty* + // answer where the interpreted class gives a full one: quiet, and wrong. the + // refusal such a class already gives is at least loud + if instance_dict(module, class) && class.fields.is_empty() { + out.push_str( + " {\"__dict__\", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL},\n", + ); + } out.push_str(" {NULL, NULL, NULL, NULL, NULL}\n};\n\n"); // the method table, using each method's python wrapper @@ -935,7 +1011,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .methods .iter() .find(|method| method.name == resume.method) - .map(|method| method.native_symbol(&module.name)) + .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); let _ = writeln!( out, @@ -1052,11 +1128,20 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ); } for method in &class.methods { + // `METH_STATIC` and `METH_CLASS` are masked off before the calling convention + // is read, so either combines with the fastcall the wrapper is written for. + // what they change is the descriptor the type publishes — a `staticmethod` or a + // `classmethod_descriptor` rather than a plain `method_descriptor` let _ = writeln!( out, - " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS, NULL}},", + " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS{}, NULL}},", method.name, - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()), + method + .binding + .method_flag() + .map(|flag| format!(" | {flag}")) + .unwrap_or_default() ); } out.push_str(" {NULL, NULL, 0, NULL}\n};\n\n"); @@ -1070,7 +1155,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .methods .iter() .find(|method| method.name == resume.method) - .map(|method| method.native_symbol(&module.name)) + .map(|method| method.native_symbol(module.name.dotted())) .unwrap_or_default(); let _ = writeln!( out, @@ -1086,7 +1171,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { .find(|field| field.name == "$kind") .map(by_ir::FieldDecl::member) .unwrap_or_default(); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let dotted = module.name.dotted(); if resume.surface == Surface::AsyncGenerator { // `__anext__` hands back an awaitable rather than an item, because the // body may `await` before it reaches its next `yield`. one `resume` @@ -1145,7 +1230,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 .am_await = {type_name}_asend_await,\n}};\n\ static PyTypeObject {type_name}_asend_type = {{\n\ \x20 PyVarObject_HEAD_INIT(NULL, 0)\n\ - \x20 .tp_name = \"{last}.{}.ascend\",\n\ + \x20 .tp_name = \"{dotted}.{}.ascend\",\n\ \x20 .tp_basicsize = sizeof({type_name}_asend),\n\ \x20 .tp_dealloc = (destructor){type_name}_asend_dealloc,\n\ \x20 .tp_flags = Py_TPFLAGS_DEFAULT,\n\ @@ -1233,13 +1318,13 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out.push_str(&emit_dunder_adapters(module, class, &type_name)); let dunders = dunder_initializers(class, &type_name); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + let dotted = module.name.dotted(); // a static struct is what a class no name reaches gets: a generator's state or a // closure's environment, neither of which anything can ask about if !heap_type(module, class) { let _ = write!( out, - "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{last}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n .tp_flags = Py_TPFLAGS_DEFAULT,\n{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ + "static PyTypeObject {type_name} = {{\n PyVarObject_HEAD_INIT(NULL, 0)\n .tp_name = \"{dotted}.{}\",\n .tp_basicsize = sizeof({struct_name}),\n .tp_itemsize = 0,\n .tp_dealloc = (destructor){type_name}_dealloc,\n .tp_flags = Py_TPFLAGS_DEFAULT,\n{iterator}{dunders} .tp_methods = {type_name}_methods,\n .tp_getset = {type_name}_getset,\n .tp_init = {type_name}_init,\n .tp_new = PyType_GenericNew,\n }};\n\ ", class.name ); @@ -1269,6 +1354,11 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { // the base happens to be collected let collected = if external_storage(module, class) { " | Py_TPFLAGS_HAVE_GC" + } else if instance_dict(module, class) { + // the dict holds whatever a decorator's generated code put in it, so the + // collector has to be able to walk it — and a managed one is only allowed on a + // type it can walk + " | Py_TPFLAGS_HAVE_GC | BY_MANAGED_DICT_FLAG" } else { "" }; @@ -1317,9 +1407,21 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { } else { format!("{init}\x20 {{Py_tp_new, (void *)PyType_GenericNew}},\n") }; + // a collected type has to hand the collector both halves, or an instance in a + // cycle is never reached at all + let walked = if instance_dict(module, class) { + format!( + "\x20 {{Py_tp_traverse, (void *){type_name}_traverse}},\n\ + \x20 {{Py_tp_clear, (void *){type_name}_clear}},\n" + ) + } else { + String::new() + }; ( format!("sizeof({struct_name})"), - format!("\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n{construction}"), + format!( + "\x20 {{Py_tp_dealloc, (void *){type_name}_dealloc}},\n{walked}{construction}" + ), ) }; let _ = write!( @@ -1330,7 +1432,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {{Py_tp_getset, (void *){type_name}_getset}},\n\ {slots}\x20 {{0, NULL}},\n}};\n\ static PyType_Spec {type_name}_spec = {{\n\ - \x20 \"{last}.{}\",\n\ + \x20 \"{dotted}.{}\",\n\ \x20 {basicsize},\n\ \x20 0,\n\ \x20 Py_TPFLAGS_DEFAULT{basetype}{collected},\n\ @@ -1342,6 +1444,53 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out } +/// whether this class or an in-module base of it carries a class decorator +/// +/// the base chain, because an instance discipline is not a per-class answer: a subclass +/// allocates and frees instances of a shape its base decided +fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { + let mut current = class; + // bounded by the class count, for the reason `inherits_layout` gives + for _ in 0..=module.classes.len() { + if !current.decorators.is_empty() { + return true; + } + match current + .base + .as_ref() + .and_then(ClassBase::in_module) + .and_then(|name| class_named(module, name)) + { + Some(next) => current = next, + None => return false, + } + } + false +} + +/// whether an emitted instance keeps a `__dict__` of its own +/// +/// a class decorator is arbitrary python handed the class, and what it hands back is +/// often code it *generated* from what it read — `@dataclass` writes an `__init__` that +/// assigns one attribute per annotation. that code is ordinary python and assumes an +/// ordinary instance, so on an emitted one, whose whole state is its layout, every +/// assignment falls off: `E(3)` raised where the interpreted class answered. the decline +/// that would otherwise be the answer is not open here — a class has no runtime fallback +/// — so the class is given the one thing the generated code needs instead. +/// +/// a managed dict costs the layout nothing: python keeps it in the pre-header, so the +/// struct, its base's prefix and every offset a compiled function reads are untouched. +/// what it costs is collection — a type with a dict of arbitrary values must be one the +/// collector walks — which is why only the classes that need it take it. +/// +/// only a class that owns its layout from `object`. one standing on a base outside the +/// module takes that base's answer about a dict, and a spec claiming one anyway would be +/// claiming room the base never allocated — which is how 24 of the `encodings` modules +/// once segfaulted +fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { + heap_type(module, class) && !inherits_layout(module, class) && decorated_chain(module, class) +} + /// whether this class takes its instance layout from a base outside the module /// /// transitively: an in-module base that itself extends an external one has no layout of @@ -1440,6 +1589,41 @@ fn appends_storage_from_a_spec(module: &ModuleIr, class: &ClassIr) -> bool { && !stands_on_an_emitted_base(module, class) } +/// the class this one appends its storage to, where that is one this module also builds +/// from a spec +/// +/// such a base is the one heap type a spec can be built on: its `tp_dealloc`, +/// `tp_traverse` and `tp_clear` are ones this module emitted, and each of those reads +/// the base to chain to from the type that *declared* it rather than from +/// `Py_TYPE(self)` — so the chain walks down to the outside base and stops, where +/// `subtype_dealloc` would come straight back. see `By_SpecSubclass`. +/// +/// the base has to come first in the module's order, because that is the order module +/// init builds them in and the subclass's spec stands on the finished type. a class +/// statement cannot name a base declared after it, so this only ever rules out a shape +/// the source could not have written. +/// +/// and the subclass has to declare only what it *adds*. the other layout model for an +/// in-module base is the struct extension, where a subclass restates its base's fields +/// so that a pointer to one is a pointer to the other — restating them in an appended +/// region instead would give the pair two copies of each, and the base's methods and the +/// subclass's would write different ones. so a field the base already stores is the +/// signal that this is the other model, and it is not appended over anything +fn appended_over_an_emitted_base<'a>(module: &'a ModuleIr, class: &ClassIr) -> Option<&'a ClassIr> { + let wanted = class.base.as_ref()?.in_module()?; + let base = module + .classes + .iter() + .take_while(|candidate| candidate.name != class.name) + .find(|candidate| candidate.name == wanted)?; + let restates = class.fields.iter().any(|field| { + base.fields + .iter() + .any(|inherited| inherited.name == field.name) + }); + (appends_storage_from_a_spec(module, base) && !restates).then_some(base) +} + /// whether this class can be built the way a `class` statement builds one — by calling /// its metaclass — rather than from a type spec /// @@ -1450,23 +1634,19 @@ fn appends_storage_from_a_spec(module: &ModuleIr, class: &ClassIr) -> bool { /// or one it inherits, since a subclass's struct begins with its base's. everything /// else a spec would have carried — the methods, and through them the type slots — /// goes in the namespace, which is where python puts it too +/// +/// a class-level constant goes in the namespace as well, so it is not a reason to keep a +/// class off this construction. what it *is* is a reason to check afterwards: the +/// namespace is where a metaclass reinterprets what the body wrote, and an `EnumType` +/// handed `STRICT = 'strict'` builds a member the module body's references do not name. +/// `By_ConstantsHeldUp` is that check, and where it fails the interpreted definition +/// stands — the same answer such a class had when this said no to it outright fn metaclass_construction(class: &ClassIr) -> bool { class.fields.is_empty() // a resumable class is a generator's state object: its state *is* its fields, // so this is already false, and nothing in the language can name it as a base && class.resume.is_none() && !decorates_a_method(class) - // a class-level constant is copied onto the finished type for the same reason a - // method decorator is applied there, and it disagrees with the metaclass the same - // way. the copy is sound under `type`, which reads nothing out of the namespace - // and leaves the value the object it already was — but a metaclass that *makes* - // something of what the body wrote never sees it. an `EnumType` handed a - // memberless namespace declares no members, and the constants land in the type's - // dict behind its back: `FlagBoundary.STRICT` answers while `_member_names_` is - // empty. feeding the twin's finished attributes into the namespace instead does - // not rescue it, because the metaclass would build *new* members and every - // reference the module body already took would still name the old ones - && class.constants.is_empty() } /// whether any of this class's methods carries a decorator @@ -1492,11 +1672,11 @@ fn class_named<'a>(module: &'a ModuleIr, name: &str) -> Option<&'a ClassIr> { /// that has always been emitted. for one appending to a base it is a lookup, because /// the offset depends on a base size known only at runtime fn fields_of(module: &ModuleIr, class: &ClassIr, object: &str) -> String { - let struct_name = class.struct_name(&module.name); + let struct_name = class.struct_name(module.name.dotted()); if external_storage(module, class) { return format!( "(({struct_name} *)By_TypeData({object}, {}_OBJ))", - class.type_name(&module.name) + class.type_name(module.name.dotted()) ); } format!("({struct_name} *){object}") @@ -1509,7 +1689,7 @@ fn fields_of(module: &ModuleIr, class: &ClassIr, object: &str) -> String { fn bind_self(module: &ModuleIr, class: &ClassIr, object: &str) -> String { format!( "{} *self = {};", - class.struct_name(&module.name), + class.struct_name(module.name.dotted()), fields_of(module, class, object) ) } @@ -1830,7 +2010,7 @@ fn on_our_operand( \x20 PyObject *by_argv[] = {{ {} }};\n\ \x20 return {}({receiver}, by_argv, {}, NULL);\n }}\n", args.join(", "), - method.wrapper_symbol(&module.name), + method.wrapper_symbol(module.name.dotted()), args.len() ) } @@ -1944,7 +2124,7 @@ fn emit_ass_subscript_adapter(module: &ModuleIr, class: &ClassIr, symbol: &str) \x20 if (by_r == NULL) return -1;\n\ \x20 Py_DECREF(by_r);\n\ \x20 return 0;\n", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ), None => format!( " PyErr_SetString(PyExc_AttributeError, {});\n\x20 return -1;\n", @@ -1984,7 +2164,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> }; let call = format!( "{}(self, NULL, 0, NULL)", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); match shape { SlotShape::Unary => { @@ -2010,7 +2190,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> "static PyObject *{symbol}(PyObject *self, PyObject *by_arg) {{\n\ \x20 PyObject *by_argv[] = {{ by_arg }};\n\ \x20 return {}(self, by_argv, 1, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } // emitted above, from both of the methods that fill this slot @@ -2023,7 +2203,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> out, "static PyObject *{symbol}(PyObject *self, PyObject *by_args, PyObject *by_kw) {{\n\ \x20 return By_CallSlot({}, self, by_args, by_kw);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::GetAttrHook => { @@ -2039,7 +2219,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 PyErr_Clear();\n\ \x20 PyObject *by_argv[] = {{ by_name }};\n\ \x20 return {}(self, by_argv, 1, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Finalize => { @@ -2068,7 +2248,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 PyObject *by_argv[] = {{ by_obj ? by_obj : Py_None,\n\ \x20 by_type ? by_type : Py_None }};\n\ \x20 return {}(self, by_argv, 2, NULL);\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Contains => { @@ -2081,7 +2261,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> \x20 int by_v = PyObject_IsTrue(by_r);\n\ \x20 Py_DECREF(by_r);\n\ \x20 return by_v;\n}}", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } SlotShape::Hash => { @@ -2125,7 +2305,7 @@ fn emit_dunder_adapters(module: &ModuleIr, class: &ClassIr, type_name: &str) -> let _ = writeln!( out, " case {opcode}: return {}(self, by_argv, 1, NULL);", - method.wrapper_symbol(&module.name) + method.wrapper_symbol(module.name.dotted()) ); } // a comparison the class does not define is not an error: answering @@ -2379,7 +2559,7 @@ fn ctype(module: &ModuleIr, ty: &RType) -> String { // a class whose fields sit past a base's instance is *not* its field struct: // the two addresses differ, and only the object pointer identifies the value Some(owner) if !external_storage(module, owner) => { - format!("{} *", owner.struct_name(&module.name)) + format!("{} *", owner.struct_name(module.name.dotted())) } _ => "PyObject *".to_string(), }, @@ -2420,7 +2600,7 @@ fn unbox_checked(module: &ModuleIr, ty: &RType, expr: &str) -> String { Some(owner) => format!( "({})By_UnboxInstance({expr}, (PyTypeObject *){}_OBJ)", ctype(module, ty), - owner.type_name(&module.name) + owner.type_name(module.name.dotted()) ), // a class with no emitted layout is represented as a plain object None => unbox_expr(ty, expr), @@ -2486,7 +2666,7 @@ fn signature(module: &ModuleIr, function: &Function) -> String { format!( "static {} {}({})", ctype(module, &function.ret), - function.native_symbol(&module.name), + function.native_symbol(module.name.dotted()), params ) } @@ -2873,6 +3053,42 @@ fn commit_checked(function: &Function, dest: RegisterId, error_target: Option String, + error_target: Option, +) -> String { + let slot = format!("by_g_{}", mangle(name)); + let mut out = format!(" {{ static PyObject *{slot} = NULL;\n"); + let _ = writeln!( + out, + " if ({slot} == NULL) {slot} = By_InternedStr({});", + c_string_sized(name) + ); + let _ = writeln!( + out, + " if ({slot} == NULL) goto {};", + error_label(error_target) + ); + out.push_str(&assign_checked( + module, + function, + dest, + &call(&slot), + error_target, + )); + out.push_str(" }\n"); + out +} + /// the tests that turn a read of an unwritten local into `UnboundLocalError` /// /// every read of a flagged register is guarded, not only the ones the analysis found @@ -3366,8 +3582,8 @@ fn emit_op( .collect::>() .join(", "); let symbol = match target { - Some(target) => target.native_symbol(&module.name), - None => format!("by_{}_{}", mangle(&module.name), mangle(callee)), + Some(target) => target.native_symbol(module.name.dotted()), + None => format!("by_{}_{}", mangle(module.name.dotted()), mangle(callee)), }; let call = format!("{symbol}({args})"); let fallible = target.is_none_or(|target| target.convention.can_fail()); @@ -3695,19 +3911,29 @@ fn emit_op( else { return String::new(); }; - let struct_name = owner.struct_name(&module.name); - let type_name = owner.type_name(&module.name); + let struct_name = owner.struct_name(module.name.dotted()); + let type_name = owner.type_name(module.name.dotted()); + // `tp_alloc` answers with the *object*, which is only the field storage for + // a class that owns its layout. one appending to a base keeps its fields + // past that base's instance, so the two addresses differ and `fields_of` is + // what knows by how much — writing through the object pointer would land on + // the base's own data let mut out = format!( " {{ PyTypeObject *by_type = (PyTypeObject *){type_name}_OBJ;\n\ - \x20 {struct_name} *by_new = ({struct_name} *)by_type->tp_alloc(by_type, 0);\n" + \x20 PyObject *by_obj = by_type->tp_alloc(by_type, 0);\n" ); // `tp_alloc` zeroes the block, so a field the loop below misses is NULL // rather than garbage — but every field is written let _ = writeln!( out, - " if (by_new == NULL) goto {};", + " if (by_obj == NULL) goto {};", error_label(error_target) ); + let _ = writeln!( + out, + " {{ {struct_name} *by_new = {};", + fields_of(module, owner, "by_obj") + ); for (field, value) in owner.fields.iter().zip(fields) { // `tp_alloc` zeroes the block, so `None` leaves the field NULL — // which is what an unset cell is @@ -3727,13 +3953,14 @@ fn emit_op( let _ = writeln!(out, " by_new->{} = 1;", field.presence()); } } + out.push_str(" }\n"); if let Some(release) = dec_ref(&RType::OBJECT, &local(*dest)) { let _ = writeln!(out, " {release}"); } let destination = function .register(*dest) .map_or_else(|| "PyObject *".to_string(), |decl| ctype(module, &decl.ty)); - let _ = writeln!(out, " {} = ({destination})by_new; }}", local(*dest)); + let _ = writeln!(out, " {} = ({destination})by_obj; }}", local(*dest)); out } Op::Enter { dest, manager } => { @@ -3832,7 +4059,7 @@ fn emit_op( else { return String::new(); }; - let table = format!("{}_methods", owner.type_name(&module.name)); + let table = format!("{}_methods", owner.type_name(module.name.dotted())); let call = format!( "By_MakeClosure(&{table}[{index}], (PyObject *)({}))", value_expr(env) @@ -3860,6 +4087,29 @@ fn emit_op( out.push_str(&commit_checked(function, *dest, error_target)); out } + // the write half of `LoadGlobal`, reaching the same dict through the same + // interned key — a register write would leave the module's binding alone + Op::StoreGlobal { dest, name, value } => global_namespace_op( + module, + function, + *dest, + name, + &|slot| { + format!( + "By_StoreGlobal(by_module_dict, {slot}, {})", + value_expr(value) + ) + }, + error_target, + ), + Op::DeleteGlobal { dest, name } => global_namespace_op( + module, + function, + *dest, + name, + &|slot| format!("By_DeleteGlobal(by_module_dict, {slot})"), + error_target, + ), Op::LoadClass { dest, class } => { let Some(owner) = module .classes @@ -3868,7 +4118,7 @@ fn emit_op( else { return String::new(); }; - let type_name = owner.type_name(&module.name); + let type_name = owner.type_name(module.name.dotted()); assign_owned( module, function, @@ -4508,7 +4758,7 @@ fn defer_tests(function: &Function, receiver: bool) -> Vec { fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> String { let mut out = format!( "static PyObject *{}(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n", - function.wrapper_symbol(&module.name) + function.wrapper_symbol(module.name.dotted()) ); // a method's receiver arrives in `self`, so it does not count as an argument if !is_method { @@ -4621,7 +4871,7 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri out, " {} by_result = {}({args});", ctype(module, &function.ret), - function.native_symbol(&module.name) + function.native_symbol(module.name.dotted()) ); if function.convention.can_fail() { let _ = writeln!( @@ -4652,22 +4902,22 @@ fn emit_wrapper(module: &ModuleIr, function: &Function, is_method: bool) -> Stri // the jump here happens before any argument local is filled, so there is // nothing to release — and no error is set, because nothing went wrong if !function.deferring.is_empty() || !function.computed_defaults.is_empty() { + // the twin is taken off the interpreted class, and for a *static* method that + // is the plain function the `staticmethod` wraps — so the call is the one a + // module-level function makes, and `self` holds nothing to put in front of it + let hands_over_self = function.owner.is_some() && function.binding != Binding::Static; let _ = writeln!(out, "by_wrap_defer: ;"); let _ = writeln!( out, " return {}({}, {}, {}args, nargs, kwnames);", - if function.owner.is_some() { + if hands_over_self { "By_CallInterpretedMethod" } else { "By_CallInterpreted" }, - function.interpreted_symbol(&module.name), + function.interpreted_symbol(module.name.dotted()), c_string(&function.name), - if function.owner.is_some() { - "self, " - } else { - "" - } + if hands_over_self { "self, " } else { "" } ); } out.push_str("}\n"); @@ -4814,6 +5064,8 @@ fn external_construction( class: &ClassIr, type_name: &str, pack: &str, + slot: Option, + twins: usize, ) -> String { let spec = if !class.keywords.is_empty() || stands_on_an_emitted_base(module, class) { // a keyword has nowhere to go in a spec, and a base of ours beside one from @@ -4827,22 +5079,68 @@ fn external_construction( } else { "by_kwds" }; + // the constants a metaclass construction writes into the namespace, and reads back off + // the class to see whether it agreed. a class no interpreted `class` statement wrote + // has no captured body to take a value off, and there is nothing to carry + let (declare, constants) = match slot.filter(|_| !class.constants.is_empty()) { + None => (String::new(), "NULL".to_string()), + Some(slot) => { + let names = class + .constants + .iter() + .map(|name| c_string(name)) + .collect::>() + .join(", "); + ( + format!( + "\x20 static const char *const by_constants[] = {{{names}}};\n\ + \x20 By_ClassConstants by_carried = {{by_body[{slot}], by_constants, {}, by_twin, by_type, {twins}}};\n", + class.constants.len() + ), + "&by_carried".to_string(), + ) + } + }; let build = format!( - "By_BuildClass(dict, {}, {pack}, {keywords}, {type_name}_methods, {spec}, {})", + "By_BuildClass(dict, {}, {pack}, {keywords}, {type_name}_methods, {spec}, {}, {constants})", c_string(&class.name), i32::from(metaclass_construction(class)) ); if class.keywords.is_empty() { - return format!("\x20 {type_name} = {build};\n"); + return format!("{declare}\x20 {type_name} = {build};\n"); } format!( - "\x20 {{ PyObject *by_kwds = {type_name}_keywords(dict);\n\ + "{declare}\ + \x20 {{ PyObject *by_kwds = {type_name}_keywords(dict);\n\ \x20 if (by_kwds == NULL) return -1;\n\ \x20 {type_name} = {build};\n\ \x20 Py_DECREF(by_kwds); }}\n" ) } +/// every decorator this module's init applies, applied where the twin left off +/// +/// the source the twin runs has these taken out of it — see +/// [`ModuleIr::decorated_at_init`](by_ir::function::ModuleIr::decorated_at_init) — so an +/// init that gives up before it has installed anything of its own still has to run them, +/// or the module is left holding definitions nothing ever decorated. it applies them to +/// the namespace entry, which on that path is still the twin's own definition — which is +/// exactly where python would have applied them +fn twin_decorators(module: &ModuleIr) -> String { + let mut out = String::new(); + for decoration in module.decorated_at_init() { + for decorator in decoration.decorators.iter().rev() { + let _ = writeln!( + out, + " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", + c_string(decoration.name), + c_string(&decorator.dotted()) + ); + } + } + out +} + fn emit_module_init(module: &ModuleIr) -> String { let mut out = String::new(); @@ -4861,26 +5159,38 @@ fn emit_module_init(module: &ModuleIr) -> String { out, " {{\"{}\", (PyCFunction)(void(*)(void)){}, METH_FASTCALL | METH_KEYWORDS, NULL}},", function.name, - function.wrapper_symbol(&module.name) + function.wrapper_symbol(module.name.dotted()) ); } } out.push_str(" {NULL, NULL, 0, NULL}\n};\n\n"); - let last = module.name.rsplit('.').next().unwrap_or(&module.name); + // the `PyModuleDef`'s `m_name`, which is not where a module's `__name__` comes + // from: this is a multi-phase init, so python builds the module from the + // *spec*'s name and never reads this one. it says the last component because + // that is what the init symbol beside it is named after + let last = module.name.last_component(); // `m_methods` is NULL and the natives are installed from the exec slot // instead, so they land *after* the interpreted definitions rather than // being overwritten by them // decorators run last: the native function has to be in the namespace before - // a decorator can be applied to it + // a decorator can be applied to it. + // + // `exported` is what says there is a namespace entry to apply one to at all — an + // unboxed edition is a second function under a mangled name nothing binds, and + // reaching for it here would fail the import with a `NameError`. it is also the + // condition `ModuleIr::decorated_at_init` states, and the twin's source has these + // decorators taken out of it on the strength of that: applying one here that the + // twin no longer applies, or the reverse, is what makes a decorator run twice or + // not at all let mut decorators = String::new(); - for function in &module.functions { + for function in module.functions.iter().filter(|function| function.exported) { for decorator in function.decorators.iter().rev() { let _ = writeln!( decorators, " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", c_string(&function.name), - c_string(decorator) + c_string(&decorator.dotted()) ); } } @@ -4892,20 +5202,83 @@ fn emit_module_init(module: &ModuleIr) -> String { // base's do. so a refusal is a whole-module one — the interpreted definition already // built the module, and it is left standing rather than made into a half-native // mixture + let mut conditions = Vec::new(); + // below 3.12 there is no way to say where appended storage goes at all, so no such + // class has a construction and the module has none either + if module + .classes + .iter() + .any(|class| appends_storage_from_a_spec(module, class)) + { + conditions.push("!BY_HAS_TYPE_DATA"); + } + // a class keeping an instance dict is the same question again: below 3.13 there is + // no published way to walk or release a managed one, and a collected type that + // cannot walk what it holds is worse than no compiled type at all + if module + .classes + .iter() + .any(|class| instance_dict(module, class)) + { + conditions.push("!BY_HAS_MANAGED_DICT"); + } + // whether the fallback source has to be run with its class bodies captured. the + // capture costs a dict copy per class the body writes, so a module with no constant + // to carry runs its body the plain way + let captures_bodies = module + .classes + .iter() + .any(|class| class.exported && !class.constants.is_empty()); + let release_bodies = if captures_bodies { + " Py_XDECREF(by_bodies);\n" + } else { + "" + }; let mut layout_guard = String::new(); + if !conditions.is_empty() { + // the twin's source no longer carries the decorators init applies, so leaving + // the module interpreted means applying them here — to the twin's own + // definitions, which is where python would have run them + let _ = write!( + layout_guard, + " if ({}) {{\n{}{release_bodies} return 0;\n }}\n", + conditions.join(" || "), + twin_decorators(module) + ); + } for class in &module.classes { if appends_storage_from_a_spec(module, class) { - let type_name = class.type_name(&module.name); - if layout_guard.is_empty() { - // below 3.12 there is no way to say where appended storage goes at all, - // so no such class has a construction and the module has none either - layout_guard.push_str(" if (!BY_HAS_TYPE_DATA) return 0;\n"); - } + let type_name = class.type_name(module.name.dotted()); + // one of these standing on another is built on the *finished* type below it + // rather than on the interpreted definition — which is the only base such a + // class can chain a deallocation to. the order is the module's, so the one + // below is already built + let construction = match appended_over_an_emitted_base(module, class) { + Some(base) => format!( + "By_SpecSubclass(dict, {}, &{type_name}_spec, {}, {}_OBJ)", + c_string(&class.name), + c_string(&base.name), + base.type_name(module.name.dotted()) + ), + None => format!( + "By_SpecClass(dict, {}, &{type_name}_spec)", + c_string(&class.name) + ), + }; + // giving up here leaves every interpreted definition standing, and their + // decorators have been taken out of the source that built them — so this + // exit has to run them for the same reason the guard above does. a module + // with nothing to run keeps the plain one-line refusal + let unwind = format!("{}{release_bodies}", twin_decorators(module)); + let refusal = if unwind.is_empty() { + "return 0;".to_string() + } else { + format!("{{\n{unwind} return 0;\n }}") + }; let _ = writeln!( layout_guard, - " {type_name} = By_SpecClass(dict, {}, &{type_name}_spec);\n\ - \x20 if ({type_name} == NULL) return 0;", - c_string(&class.name) + " {type_name} = {construction};\n\ + \x20 if ({type_name} == NULL) {refusal}" ); } } @@ -4925,9 +5298,35 @@ fn emit_module_init(module: &ModuleIr) -> String { .collect(); let mut twin_init = String::new(); let mut adopt_init = String::new(); + // and the alias remap after that, once every decorator has settled what stands under + // each class's own name — see `By_RemapTwinAliases` for why it waits that long + let mut twin_remap = String::new(); if !twins.is_empty() { let count = twins.len(); - let _ = writeln!(twin_init, " PyObject *by_twin[{count}];"); + // the types are held alongside the twins from here rather than gathered at the + // adoption, because a class constant is remapped against them as its class is + // built. a slot is NULL until then, and `By_TwinReplacement` reads that as a + // refusal — so a constant naming a class built later is left off rather than + // copied across as the twin + let _ = writeln!( + twin_init, + " PyObject *by_twin[{count}];\n\ + \x20 PyObject *by_type[{count}] = {{NULL}};" + ); + // and the bodies those `class` statements wrote, which is where a class-level + // constant's value comes from — the twin has been through its own decorators by + // now, and `By_RunModuleBody` says what that costs. borrowed from `by_bodies`, + // which is held for the whole of this function + if twins.iter().any(|class| !class.constants.is_empty()) { + let _ = writeln!(twin_init, " PyObject *by_body[{count}];"); + for (slot, class) in twins.iter().enumerate() { + let _ = writeln!( + twin_init, + " by_body[{slot}] = By_ClassBody(by_bodies, {});", + c_string(&class.name) + ); + } + } for (slot, class) in twins.iter().enumerate() { let _ = writeln!( twin_init, @@ -4937,27 +5336,44 @@ fn emit_module_init(module: &ModuleIr) -> String { } let _ = writeln!( adopt_init, - " {{ PyObject *by_type[{count}];\n\ - \x20 int by_carried;" + " if (By_AdoptTwinAttributes(by_twin, by_type, {count}) < 0) return -1;" ); - for (slot, class) in twins.iter().enumerate() { - let _ = writeln!( - adopt_init, - "\x20 by_type[{slot}] = {}_OBJ;", - class.type_name(&module.name) - ); - } + let names = twins + .iter() + .map(|class| c_string(&class.name)) + .collect::>() + .join(", "); let _ = writeln!( - adopt_init, - "\x20 by_carried = By_AdoptTwinAttributes(by_twin, by_type, {count});\n\ + twin_remap, + " {{ static const char *const by_name[] = {{{names}}};\n\ + \x20 int by_remapped = By_RemapTwinAliases(dict, by_twin, by_name, {count});\n\ \x20 for (Py_ssize_t by_at = 0; by_at < {count}; by_at++) Py_XDECREF(by_twin[by_at]);\n\ - \x20 if (by_carried < 0) return -1; }}" + \x20 if (by_remapped < 0) return -1; }}" ); } let mut class_init = String::new(); + // a class decorator is arbitrary python handed the class, and what it reads is the + // class *body* — its annotations, its class-level defaults, what its `__dict__` + // holds. none of that is on the emitted type until the twin's attributes have been + // adopted, so every decorator waits for that and runs in a pass of its own. + // + // what this costs is a class resolving an in-module base *through the namespace* — + // the mixed `class C(Base, Mixin)` shape — which now stands on the emitted type + // rather than on what `Base`'s decorator returned. that is already what a class on + // an in-module base alone does, which builds on `Base`'s type object directly + let mut class_decorate = String::new(); + // which slot of the twin arrays a class occupies. it is counted rather than looked up + // by name, because `twins` is this same walk filtered on `exported` and two classes in + // one module can be given the same name + let mut exported_so_far = 0; for class in &module.classes { - let type_name = class.type_name(&module.name); + let type_name = class.type_name(module.name.dotted()); + let slot = class.exported.then(|| { + let slot = exported_so_far; + exported_so_far += 1; + slot + }); let ready = if appends_storage_from_a_spec(module, class) { // already built, by the one construction open to it String::new() @@ -4987,7 +5403,12 @@ fn emit_module_init(module: &ModuleIr) -> String { module, class, &type_name, - &format!("PyTuple_Pack(1, {}_OBJ)", base.type_name(&module.name)) + &format!( + "PyTuple_Pack(1, {}_OBJ)", + base.type_name(module.name.dotted()) + ), + slot, + twins.len() ) ) } else if let Some(externals) = class.base.as_ref().and_then(ClassBase::external) { @@ -5033,7 +5454,7 @@ fn emit_module_init(module: &ModuleIr) -> String { {}\ \x20 {releases}\n\ \x20 if ({type_name} == NULL) return -1; }}", - external_construction(module, class, &type_name, &pack) + external_construction(module, class, &type_name, &pack, slot, twins.len()) ) } else { format!( @@ -5044,6 +5465,12 @@ fn emit_module_init(module: &ModuleIr) -> String { if !ready.is_empty() { let _ = writeln!(class_init, "{ready}"); } + // the type exists from here, so it is what stands for this class's twin in every + // remap below. a class built in the layout guard was built before the array even + // existed, which is why this is not written where the construction is + if let Some(slot) = slot { + let _ = writeln!(class_init, " by_type[{slot}] = {type_name}_OBJ;"); + } // the awaitable `__anext__` hands back is a type of its own, and an unreadied // type has no `tp_free` — `PyObject_New` on one segfaults rather than failing if class @@ -5072,11 +5499,13 @@ fn emit_module_init(module: &ModuleIr) -> String { // evaluated it at class-definition time. copying keeps the *same* object, // which is what evaluating once means for constant in &class.constants { + // a class no interpreted `class` statement wrote has no body to take one off + let Some(slot) = slot else { continue }; let _ = writeln!( class_init, - " if (By_CopyClassConstant(dict, {}, (PyTypeObject *){type_name}_OBJ, {}) < 0) return -1;", - c_string(&class.name), - c_string(constant) + " if (By_CopyClassConstant(by_body[{slot}], (PyTypeObject *){type_name}_OBJ, {}, by_twin, by_type, {}) < 0) return -1;", + c_string(constant), + twins.len() ); } // a decorated method is decorated *after* the type exists, which is the only @@ -5090,7 +5519,7 @@ fn emit_module_init(module: &ModuleIr) -> String { let names = method .decorators .iter() - .map(|decorator| c_string(decorator)) + .map(|decorator| c_string(&decorator.dotted())) .collect::>() .join(", "); let _ = writeln!( @@ -5110,14 +5539,16 @@ fn emit_module_init(module: &ModuleIr) -> String { " if (PyDict_SetItemString(dict, \"{}\", {type_name}_OBJ) < 0) return -1;", class.name ); - // and its own decorators after that, because a decorator replaces the - // namespace entry — which is where every construction looks + // and its own decorators after the adoption, because a decorator *reads* the + // class it is handed and every one of those reads has to see the body the + // `class` statement wrote. they go after the namespace entry too, because a + // decorator replaces it — which is where every construction looks for decorator in class.decorators.iter().rev() { let _ = writeln!( - class_init, + class_decorate, " if (By_ApplyDecorator(dict, {}, {}) < 0) return -1;", c_string(&class.name), - c_string(decorator) + c_string(&decorator.dotted()) ); } } @@ -5143,7 +5574,7 @@ fn emit_module_init(module: &ModuleIr) -> String { // taken while the interpreted definition is still the one under this name let mut interpreted_init = String::new(); for function in module.all_functions().filter(|function| function.defers()) { - let handle = function.interpreted_symbol(&module.name); + let handle = function.interpreted_symbol(module.name.dotted()); match &function.owner { // a method's twin is an attribute of the interpreted class, which is what // sits under the class's name until the compiled type replaces it @@ -5169,6 +5600,27 @@ fn emit_module_init(module: &ModuleIr) -> String { } } + // the body is run either way; what a class-level constant needs is what each `class` + // statement wrote *before* its own decorators, which only the capturing run keeps + let run_body = if captures_bodies { + "\x20 PyObject *by_bodies = NULL;\n\ + \x20 if (by_fallback_source[0] != '\\0') {\n\ + \x20 by_bodies = By_RunModuleBody(by_fallback_source, dict);\n\ + \x20 if (by_bodies == NULL) return -1;\n\ + \x20 }\n" + .to_string() + } else { + "\x20 if (by_fallback_source[0] != '\\0') {\n\ + \x20 if (PyDict_GetItemString(dict, \"__builtins__\") == NULL &&\n\ + \x20 PyDict_SetItemString(dict, \"__builtins__\", PyEval_GetBuiltins()) < 0) {\n\ + \x20 return -1;\n\ + \x20 }\n\ + \x20 PyObject *result = PyRun_String(by_fallback_source, Py_file_input, dict, dict);\n\ + \x20 if (result == NULL) return -1;\n\ + \x20 Py_DECREF(result);\n\ + \x20 }\n" + .to_string() + }; let _ = write!( out, "static int by_exec(PyObject *module) {{\n\ @@ -5176,20 +5628,15 @@ fn emit_module_init(module: &ModuleIr) -> String { \x20 if (dict == NULL) return -1;\n\ \x20 by_module_dict = dict;\n\ {literal_init}\ - \x20 if (by_fallback_source[0] != '\\0') {{\n\ - \x20 if (PyDict_GetItemString(dict, \"__builtins__\") == NULL &&\n\ - \x20 PyDict_SetItemString(dict, \"__builtins__\", PyEval_GetBuiltins()) < 0) {{\n\ - \x20 return -1;\n\ - \x20 }}\n\ - \x20 PyObject *result = PyRun_String(by_fallback_source, Py_file_input, dict, dict);\n\ - \x20 if (result == NULL) return -1;\n\ - \x20 Py_DECREF(result);\n\ - \x20 }}\n\ + {run_body}\ {layout_guard}\ {interpreted_init}\ {twin_init}\ {class_init}\ {adopt_init}\ + {class_decorate}\ + {twin_remap}\ + {release_bodies}\ \x20 if (PyModule_AddFunctions(module, by_methods) < 0) return -1;\n\ {decorators}\ \x20 return 0;\n\ @@ -5232,7 +5679,7 @@ mod tests { fn module_with(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), @@ -5415,6 +5862,227 @@ mod tests { assert!(refusal < install, "the refusal comes first: {c}"); } + /// a class appending storage: `tp_alloc` hands back the *object*, and its fields are + /// somewhere past it. writing them through the object pointer would land on the + /// base's own data — 8 bytes into a `complex`, say — so the construction has to + /// reach the storage the same way every other field access does + #[test] + fn constructing_a_class_that_appends_storage_writes_past_the_base() { + let mut builder = FunctionBuilder::new("make", RType::OBJECT); + let tag = builder.param("tag", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + builder.push(Op::NewInstance { + dest: made, + class: "Wrapped".to_string(), + fields: vec![Some(Value::Register(tag))], + }); + builder.terminate(Terminator::Return(Value::Register(made))); + let mut module = module_with(builder.finish()); + module.classes.push(appending_class()); + let c = emit_module(&module); + + assert!( + c.contains( + "By_app_Wrapped *by_new = ((By_app_Wrapped *)By_TypeData(by_obj, By_app_Wrapped_Type_OBJ));" + ), + "the storage is found, not assumed: {c}" + ); + assert!( + !c.contains("(By_app_Wrapped *)by_type->tp_alloc"), + "the allocation is an object, not a field struct: {c}" + ); + // and the value handed on is still the object — the field storage is not one + assert!(c.contains("r1 = (PyObject *)by_obj;"), "{c}"); + } + + /// the same construction for a class that owns its layout: the object *is* the + /// storage there, and finding it must not cost a lookup + #[test] + fn constructing_a_class_that_owns_its_layout_casts_the_object() { + let mut builder = FunctionBuilder::new("make", RType::OBJECT); + let tag = builder.param("tag", RType::OBJECT); + let made = builder.temp(RType::OBJECT); + builder.push(Op::NewInstance { + dest: made, + class: "Owned".to_string(), + fields: vec![Some(Value::Register(tag))], + }); + builder.terminate(Terminator::Return(Value::Register(made))); + let mut module = module_with(builder.finish()); + let mut owned = appending_class(); + owned.name = "Owned".to_string(); + owned.base = None; + module.classes.push(owned); + let c = emit_module(&module); + + assert!( + c.contains("By_app_Owned *by_new = (By_app_Owned *)by_obj;"), + "no lookup for a class that owns its layout: {c}" + ); + assert!(!c.contains("By_TypeData(by_obj"), "{c}"); + } + + /// two classes appending storage, one over the other. the inner one is built on the + /// *finished* type of the outer rather than on the interpreted definition, because + /// the interpreted definition's `tp_dealloc` is `subtype_dealloc` — which picks the + /// deallocator to chain to out of `Py_TYPE(self)`, finds this class's own and calls + /// it back until the stack runs out + #[test] + fn appended_storage_over_appended_storage_stands_on_the_emitted_base() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields[0].name = "depth".to_string(); + module.classes.push(inner); + let c = emit_module(&module); + + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_app_Deeper_Type_spec, \"Wrapped\", By_app_Wrapped_Type_OBJ);" + ), + "the subclass stands on the emitted base: {c}" + ); + // the base is still the one construction that has to answer from reality, and + // it is built first + let base = c + .find("By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\"") + .expect("the base is built from the twin's own bases"); + let derived = c + .find("By_app_Deeper_Type = By_SpecSubclass(") + .expect("the subclass is built after it"); + assert!(base < derived, "the base is built first: {c}"); + // and a refusal is still the whole module's + assert!( + c.contains("if (By_app_Deeper_Type == NULL) return 0;"), + "{c}" + ); + } + + /// an instance counts as one reference to its type however deep the chain of + /// appended storage is. each traverse chains to the one below, so reporting the link + /// unconditionally would report it once per rung — and a collector told an instance + /// holds two references where it holds one has been told the type is garbage + #[test] + fn the_link_to_the_type_is_reported_by_exactly_one_traverse() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let c = emit_module(&module); + assert!( + c.contains( + " if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_VISIT(Py_TYPE(self));" + ), + "the rung whose base carries it already does not: {c}" + ); + assert_eq!( + c.matches("Py_VISIT(Py_TYPE(self))").count(), + 1, + "one report per traverse: {c}" + ); + } + + /// and the same rung drops it. an instance holds one reference to its type; a chain + /// of appended storage whose every rung drops one loses the type after five + /// instances and the process goes with it + #[test] + fn the_reference_to_the_type_is_dropped_by_exactly_one_deallocator() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let c = emit_module(&module); + assert!( + c.contains( + " if (!(by_base->tp_flags & Py_TPFLAGS_HEAPTYPE)\n\ + \x20 && (by_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) Py_DECREF(by_type);" + ), + "the rung whose base drops it already does not: {c}" + ); + assert_eq!(c.matches("Py_DECREF(by_type)").count(), 1, "{c}"); + // and the base is read before the deallocation, because after it the object is + // gone and the type pointer with it + let read = c.find("by_base = ").expect("the base is read"); + let free = c + .find("by_base->tp_dealloc(self);") + .expect("and chained to"); + assert!(read < free, "{c}"); + } + + /// the other layout model for an in-module base: the subclass restates its base's + /// fields so that a pointer to one is a pointer to the other. an appended region + /// would give the pair two copies of each, so such a class is not appended over + /// anything and keeps the construction that answers from reality — which refuses + #[test] + fn a_subclass_restating_its_bases_fields_is_not_appended_over_it() { + let mut module = module_with(add()); + module.classes.push(appending_class()); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields.push(by_ir::function::FieldDecl { + name: "depth".to_string(), + ty: RType::OBJECT, + default: None, + optional: false, + }); + module.classes.push(inner); + let c = emit_module(&module); + + assert!(!c.contains("By_SpecSubclass"), "{c}"); + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_app_Deeper_Type_spec);" + ), + "{c}" + ); + } + + /// a base that lays nothing out of its own is built by calling its metaclass, so its + /// type is a `class` statement's after all — `subtype_dealloc` and the recursion + /// that comes with it. only a base built from a spec here can be chained to + #[test] + fn a_base_this_module_does_not_build_from_a_spec_is_not_one_to_stand_on() { + let mut module = module_with(add()); + let mut base = appending_class(); + base.fields.clear(); + module.classes.push(base); + let mut inner = appending_class(); + inner.name = "Deeper".to_string(); + inner.base = Some(ClassBase::InModule("Wrapped".to_string())); + inner.fields[0].name = "depth".to_string(); + module.classes.push(inner); + let c = emit_module(&module); + + assert!(!c.contains("By_SpecSubclass"), "{c}"); + assert!( + c.contains( + "By_app_Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_app_Deeper_Type_spec);" + ), + "{c}" + ); + } + + fn appending_class() -> ClassIr { + ClassIr { + name: "Wrapped".to_string(), + immutable: false, + exported: true, + base: Some(ClassBase::External(vec!["Exception".to_string()])), + inherited_init: false, + generic: false, + constants: Vec::new(), + fields: vec![by_ir::function::FieldDecl { + name: "tag".to_string(), + ty: RType::OBJECT, + default: None, + optional: false, + }], + decorators: Vec::new(), + methods: Vec::new(), + resume: None, + keywords: Vec::new(), + } + } + #[test] fn an_infallible_function_emits_no_error_path_at_all() { // this is error-path elision: the `raises Never` contract, in the C @@ -5444,7 +6112,7 @@ mod tests { caller.terminate(Terminator::Return(Value::Register(out))); let module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![callee.finish(), caller.finish()], declined: Vec::new(), classes: Vec::new(), @@ -5634,7 +6302,7 @@ mod tests { #[test] fn the_module_init_matches_the_import_name() { let module = ModuleIr { - name: "pkg.app".to_string(), + name: by_ir::ModuleName::new("pkg.app"), functions: vec![add()], classes: Vec::new(), gradual: Vec::new(), @@ -5655,6 +6323,38 @@ mod tests { )); } + /// cpython reads a class's `__module__` off the front of its `tp_name` and its + /// `__name__` off the back, so a class in a package needs the *whole* dotted + /// name there. emitting only the last component made a class in `tkinter/m.py` + /// answer `__module__ == "m"`, which `sys.modules` has nothing under + #[test] + fn a_type_in_a_package_names_the_dotted_module_it_came_from() { + let mut module = module_with(add()); + module.name = by_ir::ModuleName::new("pkg.app"); + module.classes.push(ClassIr { + name: "Point".to_string(), + immutable: false, + exported: true, + base: None, + inherited_init: false, + generic: false, + constants: Vec::new(), + fields: vec![by_ir::function::FieldDecl { + name: "x".to_string(), + ty: RType::INT, + default: None, + optional: false, + }], + decorators: Vec::new(), + methods: Vec::new(), + resume: None, + keywords: Vec::new(), + }); + let c = emit_module(&module); + assert!(c.contains("\"pkg.app.Point\""), "{c}"); + assert!(!c.contains("\"app.Point\""), "{c}"); + } + #[test] fn the_interpreted_definitions_run_before_the_natives_are_installed() { let mut module = module_with(add()); diff --git a/crates/by_ir/src/builder.rs b/crates/by_ir/src/builder.rs index e11685f515..d70f894c70 100644 --- a/crates/by_ir/src/builder.rs +++ b/crates/by_ir/src/builder.rs @@ -5,7 +5,7 @@ //! and refuses to finish a block twice, which removes the two mistakes that would //! otherwise account for most verifier failures. -use crate::function::{BasicBlock, CallConvention, Function, RegisterDecl}; +use crate::function::{BasicBlock, CallConvention, Decorator, Function, RegisterDecl}; use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; use crate::rtype::RType; @@ -17,7 +17,7 @@ pub struct FunctionBuilder { convention: CallConvention, exported: bool, owner: Option, - decorators: Vec, + decorators: Vec, registers: Vec, /// `None` until the block is sealed with a terminator blocks: Vec>, @@ -78,7 +78,7 @@ impl FunctionBuilder { } /// decorators to apply at module init, outermost first - pub fn decorators(&mut self, decorators: Vec) -> &mut Self { + pub fn decorators(&mut self, decorators: Vec) -> &mut Self { self.decorators = decorators; self } @@ -276,6 +276,7 @@ impl FunctionBuilder { range: self.range, deferring: self.deferring, computed_defaults: self.computed_defaults, + binding: crate::function::Binding::Instance, } } } diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index f2d0d8d57c..63404beb54 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -4,6 +4,8 @@ //! declared once with a type, which is what makes the representation invariant //! checkable: every write to a register must produce that register's type. +use std::path::PathBuf; + use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; use crate::rtype::RType; @@ -107,6 +109,103 @@ impl CallConvention { } } +/// a decorator expression, as much of one as module init can evaluate +/// +/// python evaluates the expression where the `def` or the `class` stands, in the +/// enclosing frame. module-level code is not compiled — it runs from the interpreted +/// twin — so there is no native frame standing there, and the only moment left is +/// module init, once that twin has run the whole body and the native definition is in +/// the namespace. +/// +/// that is faithful for an expression which is nothing but *reads*, and only for one. +/// a call would be made a second time, at the end of the module rather than where it +/// was written, and whatever it did on the way would happen twice — so `@lru_cache(512)` +/// has no variant here and declines instead +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decorator { + /// a name, or a chain of attributes read off one: `functools.cache` + /// + /// the root resolves the way `LOAD_GLOBAL` resolves it, so a decorator defined in + /// the module, imported into it, or a builtin all work; each attribute is then a + /// plain `getattr` + Path { + root: String, + attributes: Vec, + }, +} + +impl Decorator { + /// a decorator written as a bare name — which is what every language *modifier* + /// translates to + pub fn name(name: impl Into) -> Self { + Self::Path { + root: name.into(), + attributes: Vec::new(), + } + } + + /// the bare name this is, where it is one + pub fn as_name(&self) -> Option<&str> { + match self { + Self::Path { root, attributes } if attributes.is_empty() => Some(root), + Self::Path { .. } => None, + } + } + + /// the name the resolution starts from, whatever shape the rest is + pub fn root(&self) -> &str { + match self { + Self::Path { root, .. } => root, + } + } + + /// the expression as written, which is also how codegen spells it: a python + /// identifier holds no `.`, so the join is unambiguous to split again + pub fn dotted(&self) -> String { + match self { + Self::Path { root, attributes } if attributes.is_empty() => root.clone(), + Self::Path { root, attributes } => format!("{root}.{}", attributes.join(".")), + } + } +} + +/// what python leaves in slot zero when a method is called through the class +/// +/// the three conventions `type_add_methods` distinguishes, and the reason they cannot +/// be inferred from the parameter list: `def make(x)` under `@staticmethod` takes an +/// `x`, and the same source without it takes a receiver written `x`. a module-level +/// function is [`Instance`](Self::Instance) and nothing reads it there — the field +/// only means anything about a method +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Binding { + /// slot zero holds the receiver: the ordinary method + #[default] + Instance, + /// there is no receiver, and slot zero is the first written parameter + Static, + /// slot zero holds the *class* rather than an instance of it + Class, +} + +impl Binding { + /// whether the boundary is handed slot zero outside the argument vector + /// + /// true for a class method as much as an instance one: `METH_CLASS` puts the class + /// in `self`, which is the same slot in the same place + pub fn takes_slot_zero_from_self(self) -> bool { + matches!(self, Self::Instance | Self::Class) + } + + /// the `METH_` flag this convention adds to a method table entry + pub fn method_flag(self) -> Option<&'static str> { + match self { + Self::Instance => None, + Self::Static => Some("METH_STATIC"), + Self::Class => Some("METH_CLASS"), + } + } +} + /// a compiled function #[derive(Debug, Clone, PartialEq)] pub struct Function { @@ -173,11 +272,14 @@ pub struct Function { /// not the arity pub computed_defaults: Vec, /// decorators to apply, outermost first, after the native function is - /// installed in the module namespace. + /// installed in the module namespace + pub decorators: Vec, + /// what python puts in slot zero when this method is reached through the class. /// - /// each is a name resolved the way `LOAD_GLOBAL` resolves it, so a decorator - /// defined in the module or imported into it both work - pub decorators: Vec, + /// `staticmethod` and `classmethod` are honoured by the method table entry rather + /// than by [applying the decorator](Self::decorators) at init — the decorator is + /// dropped where this says anything other than [`Binding::Instance`] + pub binding: Binding, } impl Function { @@ -448,7 +550,7 @@ pub struct ClassIr { /// a construction resolves the class through that namespace, so it gets whatever /// the decorator produced — which is what makes decorating the *class* sound /// where decorating a construction site would not be - pub decorators: Vec, + pub decorators: Vec, /// methods, whose first parameter is the receiver pub methods: Vec, /// how the class resumes, when it is a generator or a coroutine @@ -544,8 +646,8 @@ impl LineTable { /// a module's worth of compiled functions #[derive(Debug, Clone, PartialEq)] pub struct ModuleIr { - /// the dotted module name, as python will import it - pub name: String, + /// the module's name — see [`ModuleName`] for why that is not one string + pub name: ModuleName, pub functions: Vec, pub classes: Vec, /// functions the compiler declined, with the reason. these fall back to the @@ -615,7 +717,65 @@ pub fn qualify(owner: Option<&str>, name: &str) -> String { } } +/// a module-level definition whose decorators module init applies to the native one +/// +/// python evaluates a decorator where the `def` or the `class` stands. the interpreted +/// twin is what stands there, so the twin evaluates it — and module init then evaluates +/// it a second time, over the native definition that replaces the twin's. the binding +/// the namespace ends up with is right either way, so what shows is not a wrong value +/// but a side effect that happened twice: a registry with two entries for one function. +/// +/// the decorator is therefore taken out of the source the twin runs, and this is the one +/// description of which ones: it drives both that removal and the C that applies them, so +/// the two cannot come to disagree. +/// +/// a module-level *function* and a module-level *class* are both here. a class the +/// module body still reaches before init has run does not get this far: it declines +/// instead, because between the twin's `class` statement and init the name holds an +/// undecorated definition, and whatever read it in that window keeps what it read +/// +/// a *method* is deliberately not here, and its decorator still runs twice. it is not +/// only a side effect that would move: the class *construction* reads what a method +/// decorator wrote, and `ABCMeta` is the case — it computes `__abstractmethods__` from +/// the namespace the body left, so taking `@abstractmethod` out of the twin empties that +/// set on every class whose construction falls back to the interpreted definition. a +/// method's decorator has to be applied where the `def` stands, which means the answer +/// is to carry the *result* across rather than to move the decorator; see the module docs +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InitDecoration<'a> { + /// the name as written, which is the name the twin's source binds it under + pub name: &'a str, + /// outermost first, the order they are written in + pub decorators: &'a [Decorator], +} + impl ModuleIr { + /// every module-level definition init decorates — see [`InitDecoration`] + pub fn decorated_at_init(&self) -> impl Iterator> { + // a closure environment is not in the namespace under any name, so there is no + // entry to apply anything to — and it carries no decorator anyway + let classes = self + .classes + .iter() + .filter(|class| class.exported && !class.decorators.is_empty()) + .map(|class| InitDecoration { + name: class.name.as_str(), + decorators: &class.decorators, + }); + // an unboxed edition is a second `Function` under a mangled name the namespace + // never holds, and it carries the decorators of the function it is an edition + // of. reaching for that name here would fail the import with a `NameError` + let functions = self + .functions + .iter() + .filter(|function| function.exported && !function.decorators.is_empty()) + .map(|function| InitDecoration { + name: function.name.as_str(), + decorators: &function.decorators, + }); + classes.chain(functions) + } + /// every compiled function, methods included /// /// a pass that iterates `functions` alone silently skips every method, which @@ -634,7 +794,7 @@ impl ModuleIr { ) } - pub fn new(name: impl Into) -> Self { + pub fn new(name: impl Into) -> Self { Self { name: name.into(), functions: Vec::new(), @@ -650,8 +810,116 @@ impl ModuleIr { /// the C identifier for the module's init function, which cpython looks up by /// name when loading the extension pub fn init_symbol(&self) -> String { - let last = self.name.rsplit('.').next().unwrap_or(&self.name); - format!("PyInit_{}", mangle(last)) + format!("PyInit_{}", mangle(self.name.last_component())) + } +} + +/// a module's name, in the forms a build needs +/// +/// they are different things, and conflating them is what made a class in +/// `tkinter/m.py` answer `__module__ == "m"`. only the dotted name is stored, so +/// they cannot drift apart: the shorter ones are derived from it on demand +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct ModuleName { + dotted: String, + /// whether the name belongs to a package's `__init__` rather than to a plain + /// module + /// + /// the dotted name cannot say, and the difference decides which file the + /// artefact is written to: `a.b` is `a/b.py` for one and `a/b/__init__.py` + /// for the other + is_package: bool, +} + +impl ModuleName { + /// a plain module — one written in a file named after its own last component + pub fn new(dotted: impl Into) -> Self { + Self { + dotted: dotted.into(), + is_package: false, + } + } + + /// a package, written in the `__init__` inside the directory the name spells + /// + /// `dotted` is the package's own name, not its `__init__`'s: `a/b/__init__.py` + /// is the module `a.b`, because that is what python imports it as and what a + /// class written in it reports as its `__module__` + pub fn package(dotted: impl Into) -> Self { + Self { + dotted: dotted.into(), + is_package: true, + } + } + + /// the name python imports the module as, dots and all + /// + /// this is what a type's `tp_name` carries: cpython splits `tp_name` at its + /// last dot to answer `__module__` and `__name__`, so a class in the module + /// `tkinter.m` needs the whole of it to say where it came from. + /// + /// it is also what every emitted C identifier is prefixed with, once the + /// mangling has turned the dots into underscores — two modules of the same + /// name in different packages then get different symbols + pub fn dotted(&self) -> &str { + &self.dotted + } + + /// the last dotted component — the module's own name inside its package + /// + /// this is cpython's rule rather than ours: cpython derives the init function + /// it looks up from the last component of the *spec* name, so an extension + /// imported as `tkinter.m` has to export `PyInit_m` however deep the package + /// goes, and a package imported as `tkinter.sub` exports `PyInit_sub` even + /// though its file is called `__init__` + /// + /// it is emphatically **not** the file name — see [`Self::relative_path`] + pub fn last_component(&self) -> &str { + self.dotted.rsplit('.').next().unwrap_or(&self.dotted) + } + + /// where the module's file belongs inside an output tree, given the suffix the + /// file carries (`".c"`, or the interpreter's extension suffix) + /// + /// the tree has to mirror the module tree, because that is the only layout + /// python will import these back under: `a.b.c` is only reachable as + /// `a/b/c`. a flat directory can offer nothing but the last component, + /// which is both unimportable as a package member and a collision — two + /// members of one package ending in the same component overwrite each other. + /// + /// a package is the case the dotted name alone gets wrong: `a.b` as a package + /// is the *directory* `a/b`, and its file is the `__init__` inside it + pub fn relative_path(&self, suffix: &str) -> PathBuf { + let mut directories: Vec<&str> = self + .dotted + .split('.') + // an empty component would contribute nothing to the path but would + // shift which component names the file, so it is dropped rather than + // joined + .filter(|component| !component.is_empty()) + .collect(); + // a package's last component names the directory holding its `__init__`; a + // plain module's names the file itself + let stem = if self.is_package { + "__init__" + } else { + directories.pop().unwrap_or_default() + }; + let mut path: PathBuf = directories.into_iter().collect(); + path.push(format!("{stem}{suffix}")); + path + } +} + +impl From<&str> for ModuleName { + fn from(dotted: &str) -> Self { + Self::new(dotted) + } +} + +impl From for ModuleName { + fn from(dotted: String) -> Self { + Self::new(dotted) } } @@ -664,6 +932,8 @@ fn mangle(name: &str) -> String { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; use crate::ops::BlockId; @@ -701,9 +971,30 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: Binding::Instance, } } + /// an unboxed edition is a second `Function` under a mangled name, carrying the + /// decorators of the function it is an edition of. the namespace never binds that + /// name, so a decorator applied to it at init would raise `NameError` and take the + /// whole import with it + #[test] + fn a_definition_the_namespace_does_not_bind_is_not_decorated_at_init() { + let mut module = ModuleIr::new("m"); + let mut exported = function(); + exported.decorators = vec![Decorator::name("mark")]; + let mut edition = exported.clone(); + edition.name = "add$arr0l".to_string(); + edition.exported = false; + module.functions = vec![exported, edition]; + let named: Vec<&str> = module + .decorated_at_init() + .map(|decoration| decoration.name) + .collect(); + assert_eq!(named, ["add"]); + } + #[test] fn symbols_are_c_identifiers() { let f = function(); @@ -713,12 +1004,73 @@ mod tests { #[test] fn the_init_symbol_uses_only_the_last_component() { - // cpython looks up PyInit_ in the loaded object + // cpython looks up PyInit_ let module = ModuleIr::new("pkg.sub.mod"); assert_eq!(module.init_symbol(), "PyInit_mod"); assert_eq!(ModuleIr::new("mod").init_symbol(), "PyInit_mod"); } + /// a package's file is called `__init__`, but cpython still derives the init + /// symbol from the last component of the name it is imported under — so the + /// symbol and the file name disagree, and only for a package + #[test] + fn a_packages_init_symbol_is_named_after_the_package_not_its_file() { + let module = ModuleIr::new(ModuleName::package("pkg.sub")); + assert_eq!(module.init_symbol(), "PyInit_sub"); + assert_eq!( + module.name.relative_path(".so"), + Path::new("pkg/sub/__init__.so") + ); + } + + /// the two halves a module name has to keep apart: what python imports the + /// module as, and the shorter name cpython's loader uses + #[test] + fn a_module_name_keeps_its_dotted_form_and_its_last_component() { + let name = ModuleName::new("pkg.sub.mod"); + assert_eq!(name.dotted(), "pkg.sub.mod"); + assert_eq!(name.last_component(), "mod"); + + // a top-level module is the same string twice, which is why the two were + // conflated for so long + let alone = ModuleName::new("mod"); + assert_eq!(alone.dotted(), "mod"); + assert_eq!(alone.last_component(), "mod"); + } + + /// the artefact tree is the module tree: a name is only importable back from + /// the path its dots spell out + #[test] + fn a_module_name_spells_out_the_path_its_artefact_is_written_to() { + assert_eq!( + ModuleName::new("pkg.sub.mod").relative_path(".c"), + Path::new("pkg/sub/mod.c") + ); + assert_eq!( + ModuleName::package("pkg.sub").relative_path(".c"), + Path::new("pkg/sub/__init__.c") + ); + assert_eq!( + ModuleName::new("mod").relative_path(".c"), + Path::new("mod.c") + ); + assert_eq!( + ModuleName::package("pkg").relative_path(".cpython-313-darwin.so"), + Path::new("pkg/__init__.cpython-313-darwin.so") + ); + } + + /// two members of one package can share a last component, and a flat output + /// directory would give them the same file — this is the collision the tree + /// layout removes + #[test] + fn two_package_members_sharing_a_last_component_take_different_paths() { + assert_ne!( + ModuleName::new("pkg.dup").relative_path(".so"), + ModuleName::new("pkg.sub.dup").relative_path(".so") + ); + } + #[test] fn params_are_the_leading_registers() { let f = function(); diff --git a/crates/by_ir/src/lib.rs b/crates/by_ir/src/lib.rs index f36d9a172d..023f9dfeeb 100644 --- a/crates/by_ir/src/lib.rs +++ b/crates/by_ir/src/lib.rs @@ -17,8 +17,8 @@ pub mod verify; pub use builder::FunctionBuilder; pub use function::{ - BasicBlock, CallConvention, ClassIr, Declined, FieldDecl, Function, GradualUse, ModuleIr, - RegisterDecl, + BasicBlock, CallConvention, ClassIr, Declined, Decorator, FieldDecl, Function, GradualUse, + ModuleIr, ModuleName, RegisterDecl, }; pub use ops::{BinOp, BlockId, CmpOp, Op, RegisterId, StandardError, Terminator, UnaryOp, Value}; pub use print::{print_function, print_module}; diff --git a/crates/by_ir/src/ops.rs b/crates/by_ir/src/ops.rs index a44d51a019..3678d1588e 100644 --- a/crates/by_ir/src/ops.rs +++ b/crates/by_ir/src/ops.rs @@ -650,6 +650,23 @@ pub enum Op { /// resolved on every read for the same reason a call is — a module global may /// be rebound, and python would see it LoadGlobal { dest: RegisterId, name: String }, + /// bind a name in the module namespace: what an assignment under a `global` + /// declaration does + /// + /// the other half of [`Self::LoadGlobal`], and it has to reach the same place. a + /// register write is private to the frame, where python's binding is the module's + /// — visible at once to every other reader, the interpreted twin included + StoreGlobal { + /// the status of the store, zero on success + dest: RegisterId, + name: String, + value: Value, + }, + /// unbind a name in the module namespace: `del x` under a `global x` + /// + /// a name that is not bound is a `NameError`, which is not what deleting from a + /// dict raises — see `By_DeleteGlobal` + DeleteGlobal { dest: RegisterId, name: String }, /// the type object of a class this module emits, by identity rather than by name /// /// a class decorator replaces the *namespace* entry, and a module may rebind the @@ -864,6 +881,8 @@ impl Op { | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } | Self::LoadGlobal { dest, .. } + | Self::StoreGlobal { dest, .. } + | Self::DeleteGlobal { dest, .. } | Self::LoadClass { dest, .. } | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } @@ -956,6 +975,8 @@ impl Op { | Self::ImportFrom { dest, .. } | Self::CallValue { dest, .. } | Self::LoadGlobal { dest, .. } + | Self::StoreGlobal { dest, .. } + | Self::DeleteGlobal { dest, .. } | Self::LoadClass { dest, .. } | Self::NewInstance { dest, .. } | Self::GetCell { dest, .. } @@ -1099,6 +1120,7 @@ impl Op { Self::SetAttr { receiver, value, .. } => vec![receiver, value], + Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } | Self::BuildList { items, .. } | Self::BuildSet { items, .. } @@ -1126,6 +1148,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::DeleteGlobal { .. } | Self::LoadClass { .. } | Self::ImportModule { .. } => Vec::new(), Self::Enter { manager, .. } => vec![manager], @@ -1266,6 +1289,7 @@ impl Op { Self::SetAttr { receiver, value, .. } => vec![receiver, value], + Self::StoreGlobal { value, .. } => vec![value], Self::TupleBuild { items, .. } | Self::BuildList { items, .. } | Self::BuildSet { items, .. } @@ -1293,6 +1317,7 @@ impl Op { Self::RaiseStandard { .. } | Self::FetchException { .. } | Self::LoadGlobal { .. } + | Self::DeleteGlobal { .. } | Self::LoadClass { .. } | Self::ImportModule { .. } => Vec::new(), Self::Enter { manager, .. } => vec![manager], diff --git a/crates/by_ir/src/print.rs b/crates/by_ir/src/print.rs index 517a21b1cc..0b7bd4a7af 100644 --- a/crates/by_ir/src/print.rs +++ b/crates/by_ir/src/print.rs @@ -11,7 +11,7 @@ use crate::ops::{Mutation, Op, RegisterId, Terminator, UnaryOp, Value}; /// render a whole module pub fn print_module(module: &ModuleIr) -> String { - let mut out = format!("module {}\n", module.name); + let mut out = format!("module {}\n", module.name.dotted()); for function in &module.functions { out.push('\n'); out.push_str(&print_function(function)); @@ -434,6 +434,14 @@ fn print_op(function: &Function, op: &Op) -> String { Op::LoadGlobal { dest, name: global } => { format!("{} = global {global}", name(*dest)) } + Op::StoreGlobal { + dest, + name: global, + value: v, + } => format!("{} = global {global} <- {}", name(*dest), value(v)), + Op::DeleteGlobal { dest, name: global } => { + format!("{} = del global {global}", name(*dest)) + } Op::LoadClass { dest, class } => { format!("{} = class {class}", name(*dest)) } @@ -699,6 +707,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, } } @@ -724,7 +733,7 @@ b2: #[test] fn a_module_lists_declines_after_its_functions() { let module = ModuleIr { - name: "app".to_string(), + name: crate::ModuleName::new("app"), functions: vec![sample()], declined: vec![Declined { range: None, @@ -764,6 +773,7 @@ b2: decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; assert!(print_function(&function).contains("return 1.0")); } diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index bea84b404e..c329901ddd 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -933,6 +933,14 @@ impl Verifier<'_> { Op::LoadGlobal { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a global read"); } + Op::StoreGlobal { dest, value, .. } => { + // the namespace holds objects, so a write to one has to arrive boxed + self.expect(block, value, &RType::OBJECT, "a global write"); + self.expect_dest(block, *dest, &RType::BIT, "a global write"); + } + Op::DeleteGlobal { dest, .. } => { + self.expect_dest(block, *dest, &RType::BIT, "a global delete"); + } Op::LoadClass { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a class read"); } @@ -1453,6 +1461,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, } } @@ -1638,6 +1647,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!( @@ -1684,6 +1694,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; assert_eq!(verify(&f), Ok(())); } @@ -1731,6 +1742,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!( @@ -1829,6 +1841,7 @@ mod tests { decorators: Vec::new(), deferring: Vec::new(), computed_defaults: Vec::new(), + binding: crate::function::Binding::Instance, }; let errors = verify(&f).unwrap_err(); assert!(errors.iter().any(|e| e.message.contains("past the end"))); @@ -1840,7 +1853,7 @@ mod tests { broken.name = "broken".to_string(); broken.ret = RType::FLOAT; let module = ModuleIr { - name: "m".to_string(), + name: crate::ModuleName::new("m"), functions: vec![add(), broken], declined: Vec::new(), classes: Vec::new(), @@ -1875,7 +1888,7 @@ mod tests { }, ); let module = ModuleIr { - name: "m".to_string(), + name: crate::ModuleName::new("m"), functions: vec![add(), caller], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_irbuild/Cargo.toml b/crates/by_irbuild/Cargo.toml index f464b30b0c..7ecf849cde 100644 --- a/crates/by_irbuild/Cargo.toml +++ b/crates/by_irbuild/Cargo.toml @@ -14,6 +14,7 @@ license = "MIT OR Apache-2.0" by_ir = { workspace = true } ruff_db = { workspace = true } ruff_python_ast = { workspace = true } +ruff_python_parser = { workspace = true } ruff_python_stdlib = { workspace = true } thin-vec = { workspace = true } ruff_text_size = { workspace = true } diff --git a/crates/by_irbuild/src/closures.rs b/crates/by_irbuild/src/closures.rs index ab30c46432..34da387f51 100644 --- a/crates/by_irbuild/src/closures.rs +++ b/crates/by_irbuild/src/closures.rs @@ -107,18 +107,6 @@ pub(crate) fn nested_functions( for (def, lambda) in definitions { let def = &def; - // a decorator on a nested function wraps the closure where the `def` - // stands, so it is resolved the way the module-level ones are: a plain - // name, looked up as `LOAD_GLOBAL` would - if def - .decorator_list - .iter() - .any(|decorator| !matches!(&decorator.expression, ast::Expr::Name(_))) - { - return Err(Decline::new( - "only a plain-name decorator on a nested function is lowered yet", - )); - } let own = own_names(def); let mut captures: Vec = Vec::new(); let mut seen: HashSet<&str> = HashSet::new(); @@ -201,6 +189,10 @@ fn own_names(def: &ast::StmtFunctionDef) -> HashSet<&str> { out.insert(kwarg.name.as_str()); } out.extend(written_names(&def.body)); + // a name declared `global` here resolves in the module namespace whether this body + // writes it or only reads it, so an enclosing local of the same name must never be + // captured for it. counting it as this function's own is what says so + out.extend(global_names(def)); // a `nonlocal` name is explicitly *not* the nested function's own for name in nonlocal_names(def) { out.remove(name); @@ -222,6 +214,18 @@ fn nonlocal_names(def: &ast::StmtFunctionDef) -> Vec<&str> { .collect() } +/// the names a function declares `global` +fn global_names(def: &ast::StmtFunctionDef) -> Vec<&str> { + crate::walk(&def.body) + .into_iter() + .filter_map(|stmt| match stmt { + Stmt::Global(node) => Some(node.names.iter().map(ruff_python_ast::Identifier::as_str)), + _ => None, + }) + .flatten() + .collect() +} + /// every name a body assigns to pub(crate) fn written_names(body: &[Stmt]) -> Vec<&str> { let mut out = Vec::new(); @@ -270,6 +274,12 @@ fn read_names(body: &[Stmt]) -> Vec<&str> { // and a lambda's body, which is an expression the walk above does reach — but a // nested `def`'s body is a statement list the walk deliberately stops at if let Stmt::FunctionDef(nested) = stmt { + // a decorator belongs to the frame the `def` stands in, not to the function + // it decorates: it is evaluated there, so the names in it are read here and + // filtered by nothing the nested function binds + for decorator in &nested.decorator_list { + collect_reads(&decorator.expression, &mut out); + } let own = own_names(nested); out.extend( read_names(&nested.body) diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index d727ddcb61..7eb9bae8ef 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -52,8 +52,8 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use by_ir::builder::FunctionBuilder; use by_ir::function::{ - CallConvention, ClassBase, ClassKeyword, Declined, Function, GradualUse, KeywordValue, - ModuleIr, qualify, + Binding, CallConvention, ClassBase, ClassKeyword, Declined, Decorator, Function, GradualUse, + KeywordValue, ModuleIr, ModuleName, qualify, }; use by_ir::ops::{ BinOp, BlockId, CmpOp, Conversion, Mutation, Op, RegisterId, StandardError, Terminator, @@ -70,12 +70,16 @@ use ty_python_semantic::ProgramEnvironment; use ty_python_semantic::{HasType, SemanticModel}; /// lower every module-level function ty can represent natively +/// +/// `module_name` is the **dotted** name python imports the module as, because +/// that is what a class in it has to answer for `__module__` — see +/// [`by_ir::ModuleName`] pub fn build_module( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, model: &SemanticModel<'_>, suite: &[Stmt], - module_name: &str, + module_name: impl Into, unique_loop_bindings: bool, ) -> ModuleIr { let mut module = ModuleIr::new(module_name); @@ -89,6 +93,51 @@ pub fn build_module( _ => None, }) .collect(); + // …and only when the *name* still holds that function. a decorator replaces what + // the module namespace binds with whatever it returned, which may be another + // function, a class, a descriptor, or no callable at all — so a call through the + // name has to go out through the namespace and find it. classes are in here for + // the same reason: a construction is written against the name, not against the + // type this module emitted under it. + // + // this is a separate set from `native_callees` because that one answers "does this + // module declare the name at all", which is what says whether `len`, `range` and + // `super` are still the builtins — and a decorator does not change that + // + // a *modifier* is not a decorator and rebinds nothing, so the question is which + // decorators survive translation rather than which definitions carry one. + // + // a name a frame declares `global` is in here for the third form of the same + // reason: the frame is going to rebind it, and the definition this module emitted + // under that name is then not what the name holds. reaching it directly answered + // with the old function and refused the new class outright + let rebinds = |decorators: &[ast::Decorator], class: bool| { + decorators.iter().any(|decorator| { + let role = if class { + class_modifier(db, model, decorator) + } else { + function_modifier(db, model, decorator) + }; + // one that declines takes its definition with it, so either answer is safe + !matches!(role, Ok(Modifier::Erased | Modifier::DataClass)) + }) + }; + let decorated: HashSet = suite + .iter() + .filter_map(|stmt| match stmt { + Stmt::FunctionDef(function) if rebinds(&function.decorator_list, false) => { + Some(function.name.to_string()) + } + Stmt::ClassDef(class) if rebinds(&class.decorator_list, true) => { + Some(class.name.to_string()) + } + _ => None, + }) + .chain(declared_global_anywhere(suite)) + .collect(); + + // every name this module reads anywhere — see `names_read` + let read = names_read(suite); // pass one: which classes get an emitted layout. a body cannot be lowered // until this is known, because whether `self.x` is a field read or a @@ -158,14 +207,13 @@ pub fn build_module( .collect(); let mut mutable: HashSet<&str> = HashSet::new(); for class in &declared { - let decorated = class.decorator_list.iter().any(|decorator| { - !matches!(&decorator.expression, Expr::Name(name) - if matches!(name.id.as_str(), "data_class" | "frozen_data_class")) - }); - if decorated { + if decorated.contains(class.name.as_str()) { mutable.insert(class.name.as_str()); } - if let Some(base) = base_class(db, env, model, class, &layouts).ok().flatten() { + if let Some(base) = base_class(db, env, model, suite, class, &layouts) + .ok() + .flatten() + { mutable.insert(class.name.as_str()); // a base of ours is made mutable too, whether it stands alone or beside a // name from outside: this class may override a method of it, and a direct @@ -327,7 +375,7 @@ pub fn build_module( }) .filter_map(|class| { // the map is the *layout* chain, which only an in-module base extends - base_class(db, env, model, class, &layouts) + base_class(db, env, model, suite, class, &layouts) .ok() .flatten() .and_then(|base| base.in_module().map(str::to_owned)) @@ -343,6 +391,8 @@ pub fn build_module( env, model, native_callees: &native_callees, + decorated: &decorated, + read: &read, suite, layouts: &layouts, methods: &methods, @@ -376,7 +426,10 @@ pub fn build_module( module .promoted .extend(promoted_places(db, env, model, function, &layouts)); - match lower_function(unit, function).and_then(verified) { + match defined_once(suite, function) + .and_then(|()| lower_function(unit, function)) + .and_then(verified) + { Ok((lowered, environments)) => { module.functions.push(lowered); // an environment is a real emitted class, just not a named one @@ -413,7 +466,8 @@ pub fn build_module( // every class the module writes, and the names its header extends. syntactic // on purpose: what the interpreted definition builds on is whatever the name // resolves to in the module namespace when the `class` statement runs, which - // is this module's compiled type wherever it emitted one + // is this module's compiled type wherever it emitted one. a name the module + // aliases stands for the class it was bound to, the same as in a base let extends: Vec<(String, Vec)> = suite .iter() .filter_map(|stmt| match stmt { @@ -423,7 +477,15 @@ pub fn build_module( .bases() .iter() .filter_map(|base| match base { - Expr::Name(name) => Some(name.id.to_string()), + Expr::Name(name) => { + // a name this cannot settle stands for itself here, which is + // what the interpreted definition looked up + Some( + base_stands_for(suite, name.id.as_str()) + .unwrap_or(name.id.as_str()) + .to_string(), + ) + } _ => None, }) .collect(), @@ -431,12 +493,24 @@ pub fn build_module( _ => None, }) .collect(); - prune_unbuildable(&mut module, &ranges, &extends, &shadowed_definitions(suite)); + prune_unbuildable( + &mut module, + &ranges, + &extends, + &disturbed_definitions(suite), + ); module } -/// every module-level `def` or `class` whose name the module body binds again -/// afterwards +/// what the module body does to its own definitions after making them +struct Disturbed { + /// every `def` or `class` whose name the body binds again + rebound: HashSet, + /// every class the body writes a dunder attribute onto, and one such name + dunder: HashMap, +} + +/// every module-level `def` or `class` the module body disturbs after defining it /// /// module init installs the native definition into the namespace over whatever the /// fallback source left there, which is the definition it replaces only while nothing @@ -450,7 +524,13 @@ pub fn build_module( /// a binding *before* the definition is the ordinary forward declaration — /// `Enum = Flag = ReprEnum = None` ahead of the classes themselves — which the /// definition then overwrites, so the two are compared by position -fn shadowed_definitions(suite: &[Stmt]) -> HashSet { +/// +/// a dunder written onto a *class* is the other half, and it is not about the name at +/// all: what an emitted type takes from its twin is the twin's own dict minus the +/// dunders, because a dunder is what a type slot answers and a second answer in the dict +/// would disagree with it. so a dunder the body wrote there has nowhere to land, and +/// where it is compared by position the ordinary forward declaration is +fn disturbed_definitions(suite: &[Stmt]) -> Disturbed { let defined: HashMap<&str, TextSize> = suite .iter() .filter_map(|stmt| match stmt { @@ -459,14 +539,80 @@ fn shadowed_definitions(suite: &[Stmt]) -> HashSet { _ => None, }) .collect(); - let mut bindings = ModuleBindings { found: Vec::new() }; + let mut bindings = ModuleBindings { + found: Vec::new(), + unbinds: false, + dunders: Vec::new(), + }; ast::visitor::walk_body(&mut bindings, suite); - bindings - .found + let classes: HashSet<&str> = suite + .iter() + .filter_map(|stmt| match stmt { + Stmt::ClassDef(class) => Some(class.name.as_str()), + _ => None, + }) + .collect(); + let dunder = bindings + .dunders .into_iter() - .filter(|(name, at)| defined.get(name).is_some_and(|defined| at > defined)) - .map(|(name, _)| name.to_string()) - .collect() + .filter(|(owner, _)| classes.contains(owner)) + .map(|(owner, attribute)| (owner.to_string(), attribute.to_string())) + .collect(); + // a name taken back out of the namespace object is a `del` whose target this cannot + // read — `ast` pops five of its own classes out through a comprehension — so every + // definition is treated as one it could have been + if bindings.unbinds { + return Disturbed { + rebound: defined.into_keys().map(str::to_string).collect(), + dunder, + }; + } + Disturbed { + rebound: bindings + .found + .into_iter() + .filter(|(name, at)| defined.get(name).is_some_and(|defined| at > defined)) + .map(|(name, _)| name.to_string()) + .collect(), + dunder, + } +} + +/// whether a name is one python spells with two underscores at each end +/// +/// the same test `By_IsDunder` makes, which is what decides at runtime whether an +/// attribute is carried from a twin +fn is_a_dunder(name: &str) -> bool { + name.len() > 4 && name.starts_with("__") && name.ends_with("__") +} + +/// whether an expression is the module namespace itself +/// +/// only the call, not a name it was stored in first: what a namespace held somewhere else +/// can be made to do is a wider question than this one, and the answer to it is not +/// syntactic +fn module_namespace(expr: &Expr) -> bool { + matches!(expr, Expr::Call(call) + if call.arguments.is_empty() + && matches!(call.func.as_ref(), Expr::Name(name) if name.id.as_str() == "globals")) +} + +/// whether an expression takes a binding back out of the module namespace +fn unbinds_through_the_namespace(expr: &Expr) -> bool { + match expr { + // `del globals()[name]`, which the store/delete context is the whole of + Expr::Subscript(subscript) => { + subscript.ctx == ExprContext::Del && module_namespace(&subscript.value) + } + Expr::Call(call) => match call.func.as_ref() { + Expr::Attribute(attribute) => { + module_namespace(&attribute.value) + && matches!(attribute.attr.as_str(), "pop" | "popitem" | "clear") + } + _ => false, + }, + _ => false, + } } /// every name the module body binds, and where @@ -476,6 +622,10 @@ fn shadowed_definitions(suite: &[Stmt]) -> HashSet { /// that bind an identifier rather than an expression are the remaining cases struct ModuleBindings<'a> { found: Vec<(&'a str, TextSize)>, + /// whether the body took a name back out of its own namespace, without saying which + unbinds: bool, + /// the dunder attributes the body writes onto a name, as `(owner, attribute)` + dunders: Vec<(&'a str, &'a str)>, } impl<'a> ast::visitor::Visitor<'a> for ModuleBindings<'a> { @@ -495,6 +645,15 @@ impl<'a> ast::visitor::Visitor<'a> for ModuleBindings<'a> { { self.found.push((name.id.as_str(), name.range.start())); } + if let Expr::Attribute(attribute) = expr + && matches!(attribute.ctx, ExprContext::Store | ExprContext::Del) + && let Expr::Name(owner) = attribute.value.as_ref() + && is_a_dunder(attribute.attr.as_str()) + { + self.dunders + .push((owner.id.as_str(), attribute.attr.as_str())); + } + self.unbinds |= unbinds_through_the_namespace(expr); ast::visitor::walk_expr(self, expr); } @@ -544,7 +703,7 @@ fn span(range: ruff_text_size::TextRange) -> (u32, u32) { fn lower_generator( unit: Unit<'_>, function: &ast::StmtFunctionDef, - decorators: Vec, + decorators: Vec, receiver: Option>, captures: Option<&closures::Nested>, ) -> Lowered<(Function, Vec)> { @@ -567,8 +726,12 @@ fn lower_generator( // a field is a *cell* — `object`, with an unset check on every read — unless the // name is definitely assigned, in which case it takes the local's own // representation and the read is an infallible `GetField` - let representations = + // a declared `global` is not one of them: it lives in the module namespace, which + // already outlives every suspension + let declared_global = declared_globals(&function.body); + let mut representations = local_representations(db, env, model, &function.body, layouts, unit.arrays); + representations.retain(|(name, _)| !declared_global.contains(name)); let locals: Vec = representations .iter() .map(|(name, _)| name.clone()) @@ -681,7 +844,7 @@ fn lower_generator_constructor( function: &ast::StmtFunctionDef, class: &str, fields: &[by_ir::function::FieldDecl], - decorators: Vec, + decorators: Vec, receiver: Option>, captured: &[String], ) -> Lowered { @@ -836,7 +999,9 @@ fn lower_resume( model, builder, locals: HashMap::new(), + globals: declared_globals(&function.body), native_callees, + decorated: unit.decorated, layouts, methods, signatures, @@ -1009,21 +1174,34 @@ fn verify_one(function: &mut Function) -> Lowered<()> { /// /// `extends` is every class the module writes and the names its header extends, /// including the ones nothing here will emit — a class left to the interpreted -/// definition still extends what it says it extends. `shadowed` is every name the -/// module body rebinds after defining it, from [`shadowed_definitions`] +/// definition still extends what it says it extends. `disturbed` is what the module body +/// does to its own definitions after making them, from [`disturbed_definitions`] fn prune_unbuildable( module: &mut ModuleIr, ranges: &HashMap, extends: &[(String, Vec)], - shadowed: &HashSet, + disturbed: &Disturbed, ) { // the name is the whole of what module init has to install under, and only an // exported definition is installed at all let rebound = |name: &str, exported: bool| { - (exported && shadowed.contains(name)).then(|| { + (exported && disturbed.rebound.contains(name)).then(|| { format!("`{name}` is rebound at module level, so installing this over it would replace what the rebind produced") }) }; + // and what the body hangs on the definition afterwards has to survive the swap too. + // `ctypes` writes `c_byte.__ctype_le__` there, and the adoption that carries a twin's + // attributes leaves every dunder behind + let hung_on = |name: &str, exported: bool| { + exported + .then(|| disturbed.dunder.get(name)) + .flatten() + .map(|attribute| { + format!( + "the module body writes `{attribute}` onto `{name}`, which the emitted type does not carry" + ) + }) + }; loop { let classes: HashSet = module .classes @@ -1036,6 +1214,7 @@ fn prune_unbuildable( .all_functions() .map(Function::qualified_name) .collect(); + let anchors = storage_anchors(module); let unbuildable = |function: &Function| -> Option { let representations = function @@ -1071,15 +1250,35 @@ fn prune_unbuildable( // a class goes as a unit: the native type object replaces the interpreted // class whole, so keeping it with one method missing would drop that // method from the module's surface - module.classes.retain(|class| { + module.classes.retain_mut(|class| { // a base this module meant to emit and did not is not a base at all, and // building on nothing in its place would quietly drop everything it brought + // + // unless this class brought no storage, and nothing under it did either. what + // stands under the base's name at import is a class either way — the + // interpreted definition, where the base declined — and building on the + // *name* is what every class over a base out of this module already does. + // + // a class with fields has no such answer: its struct begins with the base's, + // at offsets only the emitted base has. neither has one whose *subclass* + // stores something, because rebuilding this class moves the whole chain's + // layout outside the module, and that subclass's fields would go from inside + // an instance to past one — which is a construction it has no answer for + // either, and one that refuses the whole module at import rather than the + // class. `urllib.request` lost every compiled definition it had that way let declined_base = class .base .as_ref() .and_then(ClassBase::in_module) .filter(|base| !classes.contains(*base)) - .map(|base| format!("`{base}` declined, so it is not a base to build on")); + .map(str::to_owned) + .and_then(|base| { + if class.fields.is_empty() && !anchors.contains(&class.name) { + class.base = Some(ClassBase::External(vec![base])); + return None; + } + Some(format!("`{base}` declined, so it is not a base to build on")) + }); // and the other way round. a class this module does not emit is still // built — by the interpreted definition, on whatever its base name // resolves to, which is the type emitted here. that is a subclass an @@ -1095,6 +1294,7 @@ fn prune_unbuildable( }) }; match rebound(&class.name, class.exported) + .or_else(|| hung_on(&class.name, class.exported)) .or(declined_base) .or_else(interpreted_subclass) .or_else(|| class.methods.iter().find_map(&unbuildable)) @@ -1131,6 +1331,34 @@ fn prune_unbuildable( } } +/// every class an emitted one keeps storage inside an instance of +/// +/// a class's struct begins with the fields of every class this module writes above it, so +/// where any of those stops being a layout of ours the storage stops being *inside* the +/// instance and starts sitting past one. these are the classes that cannot be moved +fn storage_anchors(module: &ModuleIr) -> HashSet { + let mut anchors = HashSet::new(); + for class in &module.classes { + if class.fields.is_empty() { + continue; + } + let mut current = class; + // bounded by the class count: a base chain cannot visit one twice without being a + // cycle, and a cycle would otherwise spin here rather than settle + for _ in 0..=module.classes.len() { + let Some(base) = current.base.as_ref().and_then(ClassBase::in_module) else { + break; + }; + anchors.insert(base.to_string()); + match module.classes.iter().find(|other| other.name == base) { + Some(next) => current = next, + None => break, + } + } + } + anchors +} + /// what declined /// the places in `function` a representation was available at but for the promotion /// @@ -1250,25 +1478,8 @@ fn lower_class<'a>( // class form with a field-initializing constructor — which is exactly what a // fixed layout needs. a plain class with bare annotations has no constructor // in the interpreted build either, so compiling one would invent behaviour - let mut is_data = false; - // a real decorator is applied to the *namespace entry* after the type is - // installed, and every construction resolves the class through that namespace — - // so it gets whatever the decorator produced - let mut class_decorators = Vec::new(); - for decorator in &class.decorator_list { - match &decorator.expression { - Expr::Name(name) if matches!(name.id.as_str(), "data_class" | "frozen_data_class") => { - is_data = true; - } - Expr::Name(name) => class_decorators.push(name.id.to_string()), - _ => { - return Err(Decline::new( - "only a plain-name class decorator is lowered yet", - )); - } - } - } - let base = base_class(db, env, model, class, layouts)?; + let (is_data, class_decorators) = class_modifiers(db, model, class)?; + let base = base_class(db, env, model, suite, class, layouts)?; let fields = class_fields(db, env, model, suite, class, layouts)?; let mut lowered = Vec::new(); @@ -1401,6 +1612,23 @@ fn lower_class<'a>( method.name ))); } + // every decorator is resolved out of the module namespace at init, so + // one rooted at a name the *class body* bound is looked up somewhere it + // does not exist. `@fieldnames.setter` is the shape this turns down, and + // it turns down the whole class: two `def`s of one name would otherwise + // become two entries in one method table + if let Some(rooted) = method.decorator_list.iter().find_map(|decorator| { + let path = function_modifier(db, model, decorator).ok()?; + let Modifier::Apply(path) = path else { + return None; + }; + class_body_binds(&class.body, path.root()).then(|| path.root().to_string()) + }) { + return Err(Decline::new(format!( + "`{rooted}` is bound by the class body, and a decorator is resolved out of the module namespace at init" + ))); + } + defined_once(&class.body, method)?; let (method, produced) = lower_method(unit, method, &class.name)?; lowered.push(method); environments.extend(produced); @@ -1433,7 +1661,11 @@ fn lower_class<'a>( "`{clash}` is both a class-level constant and a field" ))); } - + // this class's own decorators are applied at init and taken out of the twin's source, + // so the module body must not reach the class in the window between the twin's + // `class` statement and init — everything it bound in that window keeps the + // definition nothing had decorated yet + decorator_stays_unread(unit.read, class.name.as_str(), &class_decorators)?; Ok(( by_ir::function::ClassIr { resume: None, @@ -1727,6 +1959,526 @@ fn attribute_of(target: &Expr, receiver: &str, owner: Option<&str>) -> Option, + class: &ast::StmtClassDef, +) -> Lowered<(bool, Vec)> { + let mut is_data = false; + let mut applied = Vec::new(); + for decorator in &class.decorator_list { + match class_modifier(db, model, decorator)? { + Modifier::DataClass => is_data = true, + Modifier::Erased => {} + Modifier::Apply(decorator) => applied.push(decorator), + } + } + if !applied.is_empty() + && let Some(unwritten) = published_beyond_the_body(class) + { + return Err(Decline::new(format!( + "an emitted type publishes `{unwritten}` alongside a method this class writes, and a decorator reads the class it is handed" + ))); + } + Ok((is_data, applied)) +} + +/// the dunders an emitted type publishes alongside this one, because they share a slot +/// +/// python reaches these through a *slot* rather than by name, and one slot backs several +/// names: `tp_richcompare` backs all six comparisons, every binary numeric slot backs an +/// operator and its reflection, and `mp_ass_subscript` backs `__setitem__` along with +/// `__delitem__`. the type publishes a wrapper for each name a filled slot backs, so a +/// class that writes one of a group gets the whole group and the rest answer +/// `NotImplemented`. +/// +/// the groups are `COMPARISONS`, `ARITHMETIC`, `POWER` and `slot_companion` in +/// `by_codegen_c`, which sits downstream of this crate and cannot be asked from here +fn shares_a_slot(name: &str) -> &'static [&'static str] { + const GROUPS: &[&[&str]] = &[ + &["__lt__", "__le__", "__eq__", "__ne__", "__gt__", "__ge__"], + &["__add__", "__radd__"], + &["__sub__", "__rsub__"], + &["__mul__", "__rmul__"], + &["__truediv__", "__rtruediv__"], + &["__floordiv__", "__rfloordiv__"], + &["__mod__", "__rmod__"], + &["__divmod__", "__rdivmod__"], + &["__lshift__", "__rlshift__"], + &["__rshift__", "__rrshift__"], + &["__and__", "__rand__"], + &["__xor__", "__rxor__"], + &["__or__", "__ror__"], + &["__matmul__", "__rmatmul__"], + &["__pow__", "__rpow__"], + &["__setitem__", "__delitem__"], + ]; + GROUPS + .iter() + .find(|group| group.contains(&name)) + .copied() + .unwrap_or(&[]) +} + +/// a name the emitted type would publish that the class body never wrote, where the +/// class writes any method at all that shares its slot with one +/// +/// this is what stops a decorator being handed a class the `class` statement did not +/// write. `@functools.total_ordering` is the shape that proves it matters: it fills in +/// the comparisons a class left out, saw `__le__` already published, added nothing — and +/// `a <= b` then raised where the interpreted class answered `True` +fn published_beyond_the_body(class: &ast::StmtClassDef) -> Option<&'static str> { + let written = |name: &str| { + class + .body + .iter() + .any(|statement| matches!(statement, Stmt::FunctionDef(method) if method.name.as_str() == name)) + }; + class.body.iter().find_map(|statement| { + let Stmt::FunctionDef(method) = statement else { + return None; + }; + shares_a_slot(method.name.as_str()) + .iter() + .copied() + .find(|name| !written(name)) + }) +} + +/// what a modifier keyword means to the native build +enum Modifier { + /// the `data class` marker, which is a layout rather than a decorator + DataClass, + /// no runtime effect at all: the transpiler erases it, so the interpreted twin + /// has nothing there either + Erased, + /// a real python decorator, applied to the finished definition at module init — + /// which is what the transpiler emits in a modifier's place + Apply(Decorator), +} + +/// the decorator expression this is, or why it is one the native build cannot evaluate +/// +/// python evaluates a decorator where the definition stands. a module-level definition +/// stands in the interpreted twin's body, which has already run by the time module init +/// installs the native one — so init is the only moment left, and the expression has to +/// be one that means the same thing there. a chain of attribute reads off a name does; a +/// call does not, because calling it at init calls it a *second* time and at the end of +/// the module rather than where it was written +fn decorator_path(expression: &Expr) -> Lowered { + let mut attributes = Vec::new(); + let mut cursor = expression; + loop { + match cursor { + Expr::Name(name) => { + attributes.reverse(); + return Ok(Decorator::Path { + root: name.id.to_string(), + attributes, + }); + } + Expr::Attribute(attribute) => { + attributes.push(attribute.attr.to_string()); + cursor = &attribute.value; + } + Expr::Call(_) => { + return Err(Decline::new( + "a decorator that is a call is evaluated where the definition stands, and module-level code is not compiled — calling it at init would run it a second time", + )); + } + _ => { + return Err(Decline::new( + "only a name, or a chain of attributes read off one, is lowered as a decorator", + )); + } + } + } +} + +/// whether a module-level definition can have its decorators moved to module init +/// +/// python runs a decorator where the definition stands, and the twin's body is what +/// stands there — so init running it again is one evaluation too many, and the decorator +/// comes out of the twin's source to make it one. that leaves a window, from the twin's +/// `def` to the end of module init, in which the name holds a definition nothing has +/// decorated yet. it is invisible unless something reads the name, and the module's own +/// body is the only thing that can: everything else runs after init. +/// +/// so this is what decides whether the move is safe, and a definition it turns down +/// declines rather than being compiled and decorated twice +fn decorator_stays_unread( + read: &BTreeSet<&str>, + name: &str, + decorators: &[Decorator], +) -> Lowered<()> { + if decorators.is_empty() || !read.contains(name) { + return Ok(()); + } + Err(Decline::new(format!( + "this module reads `{name}`, and its decorator cannot run where the definition stands and again over the compiled one" + ))) +} + +/// every name this module's own body can read before module init has finished +/// +/// a decorator module init applies is taken out of the source the twin runs — see +/// [`without_init_decorators`] — so from the twin's `def` until init reaches it, the +/// definition stands in the namespace undecorated. anything that reads the name in that +/// window keeps what it read: `TABLE = f()` in the module body straightforwardly, and +/// `def g(): return f()` called from that body just the same, because `g` reads the +/// global when it runs and not when it was written. +/// +/// a load inside a `def` is a different matter: it happens when that definition *runs*, +/// and everything that runs after import sees the decorated name. so the body reaches one +/// only by naming the definition it is in — which is itself a load the body makes, and is +/// followed from there. a module that defines helpers and calls none of them at import +/// reads nothing at all, which is the common shape and the one that keeps compiling. +/// +/// an annotation is a read only where python evaluates one. under +/// `from __future__ import annotations` it never does — `def f(x: Held)` stores the +/// *string* `"Held"` — so a module written that way names a class in a signature without +/// ever holding what the name meant at that moment +fn names_read(suite: &[Stmt]) -> BTreeSet<&str> { + /// loads the module body makes as it runs, and the loads each definition holds + /// behind its own name + #[derive(Default)] + struct Reads<'a> { + now: BTreeSet<&'a str>, + held: HashMap<&'a str, BTreeSet<&'a str>>, + } + /// a walk that files every load under one heading + struct Into<'a, 'r>(&'r mut BTreeSet<&'a str>); + impl<'a> ruff_python_ast::visitor::Visitor<'a> for Into<'a, '_> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + { + self.0.insert(name.id.as_str()); + } + ruff_python_ast::visitor::walk_expr(self, expr); + } + } + fn statements<'a>(into: &mut BTreeSet<&'a str>, body: &'a [Stmt]) { + for statement in body { + ruff_python_ast::visitor::walk_stmt(&mut Into(into), statement); + } + } + /// everything a `def` evaluates where it stands: its decorators, its defaults, and + /// its annotations where this module evaluates those. only the body waits to be called + fn header<'a>( + into: &mut BTreeSet<&'a str>, + function: &'a ast::StmtFunctionDef, + evaluated: bool, + ) { + let mut visit = Into(into); + for decorator in &function.decorator_list { + ruff_python_ast::visitor::walk_expr(&mut visit, &decorator.expression); + } + for parameter in &function.parameters { + if let Some(annotation) = parameter.annotation() + && evaluated + { + ruff_python_ast::visitor::walk_expr(&mut visit, annotation); + } + if let Some(default) = parameter.default() { + ruff_python_ast::visitor::walk_expr(&mut visit, default); + } + } + if let Some(returns) = &function.returns + && evaluated + { + ruff_python_ast::visitor::walk_expr(&mut visit, returns); + } + } + + let evaluated = annotations_are_evaluated(suite); + let mut reads = Reads::default(); + for statement in suite { + match statement { + Stmt::FunctionDef(function) => { + header(&mut reads.now, function, evaluated); + let held = reads.held.entry(function.name.as_str()).or_default(); + statements(held, &function.body); + } + // a class body runs with the module, so what it reads the module reads. its + // methods' bodies do not, and they wait behind the class's own name + Stmt::ClassDef(class) => { + let mut visit = Into(&mut reads.now); + for decorator in &class.decorator_list { + ruff_python_ast::visitor::walk_expr(&mut visit, &decorator.expression); + } + if let Some(arguments) = &class.arguments { + ruff_python_ast::visitor::walk_arguments(&mut visit, arguments); + } + for member in &class.body { + match member { + Stmt::FunctionDef(method) => { + header(&mut reads.now, method, evaluated); + let held = reads.held.entry(class.name.as_str()).or_default(); + statements(held, &method.body); + } + other => statements(&mut reads.now, std::slice::from_ref(other)), + } + } + } + other => statements(&mut reads.now, std::slice::from_ref(other)), + } + } + + // naming a definition is enough to have reached what it holds: the body may call it, + // hand it to something that calls it, or store it where a later statement will + let mut read = reads.now; + let mut pending: Vec<&str> = read.iter().copied().collect(); + while let Some(name) = pending.pop() { + let Some(held) = reads.held.get(name) else { + continue; + }; + for inner in held { + if read.insert(inner) { + pending.push(inner); + } + } + } + read +} + +/// the interpreted twin's source with every decorator module init re-applies blanked out +/// +/// a decorator is evaluated once in python, where the definition stands. the twin's `def` +/// is what stands there, and module init evaluates the same decorator a *second* time over +/// the native definition that replaces the twin's — so `@register` puts two entries in its +/// registry and `@count_them` counts one function twice. the binding the namespace ends up +/// with is right either way, which is what makes this a silent one. +/// +/// only the decorators [`ModuleIr::decorated_at_init`] names are removed, and each is +/// matched by the path it was written as, so a decorator init does *not* re-apply — the +/// `@dataclass` a `data class` becomes, the `@staticmethod` the method table honours +/// instead, a *method's* which init applies to the finished type — is left where it is +/// and still runs once. +/// +/// they are blanked rather than cut, because a traceback through the twin quotes its +/// source by line: taking the line out would move every definition below it. +/// +/// this is deliberately keyed off the *twin's* text rather than the original source's, +/// because the twin is what runs. for a `.by` module the two are not the same file — the +/// twin is the transpiler's output, where a modifier has already become the decorator it +/// stands for +pub fn without_init_decorators(source: &str, module: &ModuleIr) -> Result { + let mut wanted: HashMap<&str, Vec<&Decorator>> = HashMap::new(); + for decoration in module.decorated_at_init() { + wanted + .entry(decoration.name) + .or_default() + .extend(decoration.decorators); + } + if wanted.is_empty() { + return Ok(source.to_string()); + } + let parsed = ruff_python_parser::parse_module(source) + .map_err(|error| format!("the interpreted fallback does not parse: {error}"))?; + + let mut blank: Vec<(usize, usize)> = Vec::new(); + let mut mark = |paths: Option<&Vec<&Decorator>>, written: &[ast::Decorator]| { + let Some(paths) = paths else { + return; + }; + let mut taken = vec![false; written.len()]; + for path in paths { + // the first unclaimed one that was written as this path. a definition may + // carry the same decorator twice, and then init re-applies it twice too — + // so each application claims one occurrence rather than all of them + let found = written.iter().enumerate().position(|(index, decorator)| { + !taken[index] + && decorator_path(&decorator.expression).is_ok_and(|found| found == **path) + }); + if let Some(index) = found { + taken[index] = true; + let range = written[index].range(); + blank.push((range.start().to_usize(), range.end().to_usize())); + } + } + }; + // only a module-level definition is ever named here, so descending further could + // only blank a decorator on an unrelated definition that happens to share a name + for statement in parsed.suite() { + match statement { + Stmt::FunctionDef(function) => { + mark(wanted.get(function.name.as_str()), &function.decorator_list); + } + Stmt::ClassDef(class) => { + mark(wanted.get(class.name.as_str()), &class.decorator_list); + } + _ => {} + } + } + + let mut out = source.as_bytes().to_vec(); + for (start, end) in blank { + for byte in &mut out[start..end] { + // a decorator may be written over several lines, and the line breaks inside + // it are what keep everything below on the line it was on + if *byte != b'\n' && *byte != b'\r' { + *byte = b' '; + } + } + } + String::from_utf8(out) + .map_err(|error| format!("blanking a decorator split a character: {error}")) +} + +/// whether this definition is the only one of its name in the scope it stands in +/// +/// two `def`s of one name bind whichever one *ran*, so a direct call cannot know which +/// function it is calling — and they mangle to one C symbol besides, which makes the +/// whole module fail to build rather than answer wrongly. `closures::plan` asks this of +/// every nested scope; the module scope had nobody asking, and +/// `importlib/resources/_common.py` has three module-level `def _`s +fn defined_once(scope: &[Stmt], function: &ast::StmtFunctionDef) -> Lowered<()> { + let named = scope + .iter() + .filter( + |statement| matches!(statement, Stmt::FunctionDef(other) if other.name.as_str() == function.name.as_str()), + ) + .count(); + if named > 1 { + return Err(Decline::new(format!( + "`{}` is defined more than once in this scope, so a call to it has no single target", + function.name + ))); + } + Ok(()) +} + +/// whether a class body binds this name +/// +/// a decorator is resolved out of the *module* namespace at module init, and a class +/// body is not that namespace. `@fieldnames.setter` reads the property the body bound +/// two statements up, which is nowhere init can look — so the method keeps its +/// interpreted definition, and with it the whole class +fn class_body_binds(body: &[Stmt], name: &str) -> bool { + body.iter().any(|statement| match statement { + Stmt::FunctionDef(node) => node.name.as_str() == name, + Stmt::ClassDef(node) => node.name.as_str() == name, + Stmt::Assign(node) => node.targets.iter().any(|target| binds_name(target, name)), + Stmt::AnnAssign(node) => binds_name(&node.target, name), + Stmt::AugAssign(node) => binds_name(&node.target, name), + _ => false, + }) +} + +/// whether this module evaluates the annotations it writes +/// +/// `from __future__ import annotations` makes every one of them a string that nothing +/// evaluates until something asks — so naming a class in a signature is not a *read* of +/// it, and a module written that way keeps compiling classes a module without it would +/// have to turn down +fn annotations_are_evaluated(suite: &[Stmt]) -> bool { + !suite.iter().any(|statement| { + matches!(statement, Stmt::ImportFrom(import) + if import.module.as_ref().is_some_and(|module| module.as_str() == "__future__") + && import.names.iter().any(|alias| alias.name.as_str() == "annotations")) + }) +} + +/// whether this decorator was written with an `@` +/// +/// basedpython's class and function *modifiers* — `sealed`, `static`, `data class` and +/// the rest — reach the ast as decorators with no `@` in front of them, which is the +/// only thing that tells the two apart. the transpiler settles them by the same test. +/// +/// this matters more here than anywhere: a decorator becomes a **name looked up in the +/// module namespace at init**, and a modifier has no such name. `static def m` compiled +/// to `By_ApplyDecorator(dict, "m", "static")` and the whole extension then failed to +/// import with `NameError: name 'static' is not defined` +fn is_written_decorator( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> bool { + let source = ruff_db::source::source_text(db, model.file()); + source + .as_bytes() + .get(usize::from(decorator.range().start())) + .copied() + == Some(b'@') +} + +/// what a class decorator means to the native build +/// +/// a written `@` decorator is applied to the *namespace entry* after the type is +/// installed, exactly as the class statement would have; [`decorator_path`] says which +/// expressions still mean there what they meant where the `class` stood. a *modifier* is +/// a bare name and never anything else, which is why the two questions are asked in this +/// order +fn class_modifier( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> Lowered { + if is_written_decorator(db, model, decorator) { + return decorator_path(&decorator.expression).map(Modifier::Apply); + } + let Expr::Name(name) = &decorator.expression else { + return Err(Decline::new( + "only a plain-name class modifier is lowered yet", + )); + }; + match name.id.as_str() { + "data_class" | "frozen_data_class" => Ok(Modifier::DataClass), + // erased by the transpiler, so the interpreted twin carries nothing either. + // `sealed` grows a `__sealed_members__` tuple, but the module body the + // fallback runs is what writes it + "abstract" | "open" | "sealed" | "export" => Ok(Modifier::Erased), + // `final` becomes `@final` from `typing`, which returns its argument — the + // one class decorator whose result is provably the class it was handed + "final" => Ok(Modifier::Apply(Decorator::name("final"))), + // `private` renames the class and `protocol` rewrites its bases: neither is a + // decorator at all, and the emitted type would answer to the wrong name or + // stand outside the protocol it was declared to be + _ => Err(Decline::new( + "this class modifier changes what the class is, which an emitted type cannot follow", + )), + } +} + +/// the same for a function or a method +/// +/// the modifier mapping is the transpiler's: each of these becomes the named python +/// decorator in the interpreted twin, and the fallback preamble is what binds the name — +/// so looking it up in the module namespace at init finds exactly what the twin used +fn function_modifier( + db: &dyn ty_python_semantic::Db, + model: &SemanticModel<'_>, + decorator: &ast::Decorator, +) -> Lowered { + if is_written_decorator(db, model, decorator) { + return decorator_path(&decorator.expression).map(Modifier::Apply); + } + let Expr::Name(name) = &decorator.expression else { + return Err(Decline::new("only a plain-name modifier is lowered yet")); + }; + match name.id.as_str() { + "static" => Ok(Modifier::Apply(Decorator::name("staticmethod"))), + "classmethod" => Ok(Modifier::Apply(Decorator::name("classmethod"))), + "abstract" => Ok(Modifier::Apply(Decorator::name("abstractmethod"))), + "final" => Ok(Modifier::Apply(Decorator::name("final"))), + "override" => Ok(Modifier::Apply(Decorator::name("override"))), + // neither reaches the interpreted twin as a decorator + "open" | "export" => Ok(Modifier::Erased), + // `private` mangles the name the definition is bound under, which is a + // rename rather than a decorator + _ => Err(Decline::new( + "this modifier changes what the definition is bound as, not what it is", + )), + } +} + /// the class a class extends /// /// a name this module does not emit is still lowerable, however many of them: the type @@ -1741,6 +2493,7 @@ fn base_class( db: &dyn ty_python_semantic::Db, env: &ProgramEnvironment<'_>, model: &SemanticModel<'_>, + suite: &[Stmt], class: &ast::StmtClassDef, layouts: &Layouts, ) -> Lowered> { @@ -1752,15 +2505,24 @@ fn base_class( // the bases external even when none were written: python supplies `(object,)` for an // empty one itself, the same as `type("C", (), ns)` does let keyed = !arguments.keywords.is_empty(); - match arguments.args.as_ref() { - [] if keyed => Ok(Some(ClassBase::External(Vec::new()))), - [] => Ok(None), + // what each base written as a plain name stands for — see `base_stands_for` + let named: Vec> = arguments + .args + .iter() + .map(|base| match base { + Expr::Name(name) => base_stands_for(suite, name.id.as_str()).map(Some), + _ => Ok(None), + }) + .collect::>>()?; + match (arguments.args.as_ref(), named.as_slice()) { + ([], _) if keyed => Ok(Some(ClassBase::External(Vec::new()))), + ([], _) => Ok(None), // `class C(object)` is what `class C:` already is — the base adds no storage // and no members, so there is nothing to lay out and nothing to inherit. // resolved rather than matched by name, because a module may bind `object` // to something else entirely - [base] if !keyed && is_builtin_object(db, env, model, base, layouts) => Ok(None), - [Expr::Name(name)] if layouts.contains_key(name.id.as_str()) => { + ([base], _) if !keyed && is_builtin_object(db, env, model, base, layouts) => Ok(None), + ([Expr::Name(_)], [Some(name)]) if layouts.contains_key(*name) => { if keyed { // the layout would have to be ours, which only the type spec lays out, // and a spec has nowhere to put a keyword @@ -1768,29 +2530,33 @@ fn base_class( "a class keyword on a base this module emits is not lowered yet", )); } - Ok(Some(ClassBase::InModule(name.id.to_string()))) + Ok(Some(ClassBase::InModule((*name).to_string()))) } // more than one base: python works out the mro and which of them owns the // layout, and this class declares none of its own. one of *ours* may stand // among them so long as it lays nothing out — it is in the module namespace by // the time this class is built, so it resolves like any other name, and having // no fields it asks for no room this class does not control - bases => { + (bases, named) => { let mut paths = Vec::with_capacity(bases.len()); - for base in bases { - if let Expr::Name(name) = base - && layouts - .get(name.id.as_str()) - .is_some_and(|fields| !fields.is_empty()) + for (base, name) in bases.iter().zip(named) { + if let Some(name) = name + && layouts.get(*name).is_some_and(|fields| !fields.is_empty()) { return Err(Decline::new( "a base this module lays out cannot stand beside one it does not", )); } - let Some(path) = dotted_path(base) else { - return Err(Decline::new( - "only a name or a dotted name is lowered as a base class yet", - )); + let path = match name { + Some(name) => (*name).to_string(), + None => match dotted_path(base) { + Some(path) => path, + None => { + return Err(Decline::new( + "only a name or a dotted name is lowered as a base class yet", + )); + } + }, }; if !external_base_resolves(model, base) { return Err(Decline::new( @@ -1804,6 +2570,109 @@ fn base_class( } } +/// the name a base written as a plain name stands for in this module's own namespace +/// +/// every question asked about a base is asked of the *name*: whether this module lays it +/// out, whether it stands beside one that does, and what the emitted module looks up at +/// import. a module-level alias answers all three about a name that is not the class. +/// +/// an emitted type is put in the namespace under the class's **own** name as it is +/// built, and an alias is carried over to it only once every class has been built — so +/// `Alias = Root` left `class C(Alias)` built on the interpreted definition while +/// `m.Root` was the emitted type, and `isinstance(C(), Root)` answered `False` where the +/// interpreter says `True`. the layout gates missed it for the same reason: +/// `class C(Alias, ABC)` compiled where `class C(Root, ABC)` is refused. +/// +/// only a name the module body binds exactly once, to another plain name, is followed, +/// and only as far as a class this module writes. a chain that leaves the module is left +/// where it was written: swapping one name the body may rebind for another buys nothing, +/// and both stand for the same object at import. +/// +/// a name bound twice stands for whichever binding ran last rather than the one the +/// class statement saw, which is a question about order that a name cannot answer — so +/// where any of those bindings is a class this module writes, the class declines +fn base_stands_for<'a>(suite: &'a [Stmt], written: &'a str) -> Lowered<&'a str> { + let mut current = written; + // bounded the way the base walks are: an alias chain cannot reach a name twice + // without being a cycle, and a cycle would otherwise spin here rather than settle + for _ in 0..=suite.len() { + if class_written(suite, current).is_some() { + return Ok(current); + } + match module_binding(suite, current) { + Bound::Loose => return Ok(written), + Bound::Alias(next) => current = next, + Bound::Contested => { + return Err(Decline::new( + "a base the module binds more than once stands for the class bound last, not the one it was built on", + )); + } + } + } + Ok(written) +} + +/// what this module's own body binds a base name to +enum Bound<'a> { + /// nothing here says, so the name stands for itself — an import, or a value no + /// class of this module's is behind + Loose, + /// the one module-level `name = other` + Alias(&'a str), + /// bound more than once, and a class this module writes is one of them + Contested, +} + +fn module_binding<'a>(suite: &'a [Stmt], name: &str) -> Bound<'a> { + let mut aliases = Vec::new(); + let mut bindings = 0usize; + for statement in suite { + let value = match statement { + Stmt::Assign(assign) => match assign.targets.as_slice() { + [Expr::Name(target)] if target.id.as_str() == name => Some(assign.value.as_ref()), + // a name among several targets is bound here too, so it counts as a + // binding even though this one does not say what to + targets if targets.iter().any(|target| binds_name(target, name)) => None, + _ => continue, + }, + // an annotation with no value binds nothing at all + Stmt::AnnAssign(assign) => match assign.target.as_ref() { + Expr::Name(target) if target.id.as_str() == name => match &assign.value { + Some(value) => Some(value.as_ref()), + None => continue, + }, + _ => continue, + }, + _ => continue, + }; + bindings += 1; + if let Some(Expr::Name(value)) = value { + aliases.push(value.id.as_str()); + } + } + match (bindings, aliases.as_slice()) { + (1, [alias]) => Bound::Alias(alias), + _ if aliases + .iter() + .any(|alias| class_written(suite, alias).is_some()) => + { + Bound::Contested + } + _ => Bound::Loose, + } +} + +/// whether an assignment target binds this name anywhere inside it +fn binds_name(target: &Expr, name: &str) -> bool { + match target { + Expr::Name(target) => target.id.as_str() == name, + Expr::Tuple(tuple) => tuple.iter().any(|element| binds_name(element, name)), + Expr::List(list) => list.iter().any(|element| binds_name(element, name)), + Expr::Starred(starred) => binds_name(&starred.value, name), + _ => false, + } +} + /// the keyword arguments a class header carries, as the emitter needs them /// /// the values are evaluated in the module scope at class-definition time, which at @@ -1989,25 +2858,8 @@ fn class_fields( "a `setattr` on the receiver names its attribute at runtime", )); } - let mut is_data = false; - // a real decorator is applied to the *namespace entry* after the type is - // installed, and every construction resolves the class through that namespace — - // so it gets whatever the decorator produced - let mut class_decorators = Vec::new(); - for decorator in &class.decorator_list { - match &decorator.expression { - Expr::Name(name) if matches!(name.id.as_str(), "data_class" | "frozen_data_class") => { - is_data = true; - } - Expr::Name(name) => class_decorators.push(name.id.to_string()), - _ => { - return Err(Decline::new( - "only a plain-name class decorator is lowered yet", - )); - } - } - } - let base = base_class(db, env, model, class, layouts)?; + let (is_data, _) = class_modifiers(db, model, class)?; + let base = base_class(db, env, model, suite, class, layouts)?; // a subclass's struct *begins* with its base's fields, in the same order and // unchanged, so a pointer to one is a valid pointer to the other — which is what @@ -2022,6 +2874,9 @@ fn class_fields( .flatten() .cloned() .collect(); + // the inherited ones come first and nothing after that removes or reorders one, so + // what this class adds of its own is whatever the field passes left past them + let taken = inherited.len(); let fields = if is_data { data_fields(db, env, model, class, layouts, inherited)? } else { @@ -2033,9 +2888,19 @@ fn class_fields( init_fields(db, env, model, class, layouts, inherited)?, )? }; - let fields = - spec_built_where_needed(db, env, model, suite, class, base.as_ref(), layouts, fields)?; - metaclass_carries_the_body(class, base.as_ref(), layouts, is_data)?; + // a class that adds no field of its own keeps what its base keeps, at the offsets + // the base laid them out, reached through the descriptors the base published — so + // there is no region past the base's instance for it to own, and none of the three + // slots that would reach one. it is built the way any other class with no storage of + // its own is, and what it declares here is the same nothing + let fields = if fields.len() == taken + && appends_past_a_base_of_ours(db, env, model, suite, base.as_ref(), layouts) + { + Vec::new() + } else { + spec_built_where_needed(db, env, model, suite, class, base.as_ref(), layouts, fields)? + }; + metaclass_carries_the_body(class, base.as_ref(), layouts)?; Ok(presence_where_a_finalizer_reads( db, env, model, suite, layouts, class, fields, )) @@ -2050,6 +2915,11 @@ fn class_fields( /// finished type, could have run. an `abstractmethod` there also raises, since a compiled /// method is a descriptor and takes no attributes. /// +/// a class-level constant is not in the same position, though it reads like it: it goes +/// into the namespace with the methods, and the class is asked afterwards whether it kept +/// the value — see `By_ConstantsHeldUp`. a decorator has no such answer, because what it +/// produces is only knowable by running it on a class that already exists. +/// /// this is asked while the layouts are still settling rather than while the body is /// lowered, and where it turns a class down that class leaves the layout set — so a /// subclass of one takes the same external base every other declining class's subclass @@ -2060,7 +2930,6 @@ fn metaclass_carries_the_body( class: &ast::StmtClassDef, base: Option<&ClassBase>, layouts: &Layouts, - is_data: bool, ) -> Lowered<()> { if class_keywords(class)?.is_empty() && !stands_on_an_emitted_base(base, layouts) { return Ok(()); @@ -2073,23 +2942,6 @@ fn metaclass_carries_the_body( "a decorated method on a class built through its metaclass is not lowered yet", )); } - // a class-level constant is settled after the metaclass has decided too, so a class - // with one is kept off that construction — and there is nowhere else for such a class - // to go. what would be left is the interpreted definition, which is only there while - // the module still holds the name: `ast` pops `Num` out of its own globals, so the - // class has to be declined here rather than left to fail at import. - // in a `data class` the annotations are the fields instead, and the layout has taken - // them - let constant = class.body.iter().any(|statement| match statement { - Stmt::AnnAssign(node) => !is_data && node.value.is_some(), - Stmt::Assign(_) => true, - _ => false, - }); - if constant { - return Err(Decline::new( - "a class-level constant on a class built through its metaclass is not lowered yet", - )); - } Ok(()) } @@ -2168,25 +3020,7 @@ fn spec_built_where_needed( "a class with fields of its own cannot have a base this module emits beside one it does not", )); } - // this class's fields sit past the base's instance, and reaching them takes three - // type slots of its own that call the base's. python's own three resolve which base - // to chain to from the instance's type rather than from the type that declared them, - // so they find this class's back and call it — a recursion that ends as a stack - // overflow. a class this module *writes* carries exactly those whichever way it is - // built: emitted from a spec, or left to the interpreted definition where it declined - let appended = match base { - None => false, - Some(ClassBase::External(_)) => true, - Some(ClassBase::InModule(name)) => { - laid_out_from_outside(db, env, model, suite, layouts, name) - } - }; - if appended - && base.is_some_and(|base| { - base.plain_names() - .any(|name| class_written(suite, name).is_some()) - }) - { + if appends_past_a_base_of_ours(db, env, model, suite, base, layouts) { return Err(Decline::new( "a class whose fields sit past a base's instance needs a base python frees itself, and one this module writes is not", )); @@ -2207,6 +3041,40 @@ fn spec_built_where_needed( Ok(fields) } +/// whether a class with storage of its own would keep it past an instance of a base this +/// module writes +/// +/// reaching such storage takes three type slots of this class's own that call the base's. +/// python's own three resolve which base to chain to from the instance's type rather than +/// from the type that declared them, so they find this class's back and call it — a +/// recursion that ends as a stack overflow. a class this module *writes* carries exactly +/// those whichever way it is built: emitted from a spec, or left to the interpreted +/// definition where it declined. +/// +/// a class that adds no field of its own asks nothing of this: there is no region past +/// the base's instance for it to own, so none of the three slots is its to supply +fn appends_past_a_base_of_ours( + db: &dyn ty_python_semantic::Db, + env: &ProgramEnvironment<'_>, + model: &SemanticModel<'_>, + suite: &[Stmt], + base: Option<&ClassBase>, + layouts: &Layouts, +) -> bool { + let appended = match base { + None => false, + Some(ClassBase::External(_)) => true, + Some(ClassBase::InModule(name)) => { + laid_out_from_outside(db, env, model, suite, layouts, name) + } + }; + appended + && base.is_some_and(|base| { + base.plain_names() + .any(|name| class_written(suite, name).is_some()) + }) +} + /// the class statement this module writes under a name, where it writes one /// /// asked of the *source* rather than of the emitted set, because what a base name stands @@ -2301,7 +3169,7 @@ fn layout_root( let mut current = class; // bounded the way [`laid_out_from_outside`] is for _ in 0..=suite.len() { - let next = base_class(db, env, model, current, layouts) + let next = base_class(db, env, model, suite, current, layouts) .ok() .flatten() .and_then(|base| base.in_module().map(str::to_owned)) @@ -2336,7 +3204,10 @@ fn laid_out_from_outside( // bounded by the class count: a base chain cannot visit one twice without being a // cycle, and a cycle would otherwise spin here rather than settle for _ in 0..=suite.len() { - match base_class(db, env, model, current, layouts).ok().flatten() { + match base_class(db, env, model, suite, current, layouts) + .ok() + .flatten() + { None => return false, Some(ClassBase::External(_)) => return true, Some(ClassBase::InModule(base)) => match class_written(suite, &base) { @@ -2631,9 +3502,11 @@ fn zero_argument_super( "python makes this method implicitly static or class, so slot zero holds the class rather than a receiver", ); } + // `static` and `classmethod` are how basedpython spells the first two, so the + // marker forms have to be read here as well as the `@` forms let rebinds_slot_zero = function.decorator_list.iter().any(|decorator| { matches!(&decorator.expression, Expr::Name(name) - if matches!(name.id.as_str(), "classmethod" | "staticmethod")) + if matches!(name.id.as_str(), "classmethod" | "staticmethod" | "static")) }); if rebinds_slot_zero { return Err( @@ -2649,6 +3522,64 @@ fn zero_argument_super( }) } +/// which of python's three method conventions this definition asks for, with the +/// decorator that asked for it taken off the list +/// +/// `staticmethod` and `classmethod` are not applied at module init like every other +/// decorator: the method table entry carries `METH_STATIC` or `METH_CLASS`, and the +/// type builds the descriptor python would have built. so honouring one means dropping +/// it, and the two must not both happen. +/// +/// that only holds where it is the **only** decorator. the runtime folds the rest onto +/// the attribute it reads back off the finished type — and reading a static method back +/// hands over the plain function, which would then be written back as an ordinary +/// method. a second decorator keeps the decline +fn method_binding( + function: &ast::StmtFunctionDef, + receiver: Option>, + decorators: &mut Vec, +) -> Lowered { + if !matches!(receiver, Some(Receiver::Explicit(_))) { + return Ok(Binding::Instance); + } + // only a bare name says which convention: `abc.abstractmethod` is not one however + // its last segment reads, and neither is any other attribute off something else + let convention = |decorator: &Decorator| match decorator.as_name()? { + "staticmethod" => Some(Binding::Static), + "classmethod" => Some(Binding::Class), + _ => None, + }; + let Some(binding) = decorators.iter().find_map(convention) else { + return Ok(Binding::Instance); + }; + if decorators.len() > 1 { + return Err(Decline::new( + "a second decorator over `classmethod` or `staticmethod` is folded onto the attribute read back off the type, which is no longer the one it was", + )); + } + // python already makes these implicitly static or class, and the emitted type + // publishes its own `__class_getitem__` for a generic class — so a table entry of + // our own would either double the convention or collide with that one + if matches!( + function.name.as_str(), + "__new__" | "__init_subclass__" | "__class_getitem__" + ) { + return Err(Decline::new( + "python gives this method a convention of its own, which a method table entry would duplicate", + )); + } + // a generator's state object is namespaced by the receiver's class, and neither of + // these has one — so two classes with a static `values` would want one state class + // between them + if generators::is_generator(&function.body) || function.is_async { + return Err(Decline::new( + "a `classmethod` or `staticmethod` that suspends is not lowered yet", + )); + } + decorators.clear(); + Ok(binding) +} + /// a method: an ordinary function whose exported name is namespaced by the class fn lower_method( unit: Unit<'_>, @@ -2724,16 +3655,50 @@ fn lower_function_with_receiver( signatures, .. } = unit; - // a decorator is applied at module init to the installed native function, so - // the body still compiles. only a plain name is supported: a call or an - // attribute would need arguments evaluated at init + // a name this frame declares `global` gets no register and no environment field: + // both halves of it are the module namespace, reached through `Place::Global`. + // keeping it out of the locals here is what makes that true — a register declared + // for it would be what a nested function captured, and the two would disagree + let declared_global = declared_globals(&function.body); + // a decorator is applied at module init to the installed native function, so the + // body still compiles. `decorator_path` says which expressions mean the same thing + // evaluated there as they did where the `def` stood. + // + // a *modifier* is not a name at all — it is spelled without an `@` and the + // transpiler rewrites it — so it is translated to whatever the interpreted twin + // ended up with, or dropped where the twin has nothing + // a *nested* function's decorators belong to the frame the `def` stands in, which + // applies them to the closure it just made — see `nested_def`. carrying them here + // as well would apply them a second time, to the environment class's method + let nested = matches!(receiver, Some(Receiver::Implicit(_))); let mut decorators = Vec::with_capacity(function.decorator_list.len()); - for decorator in &function.decorator_list { - let Expr::Name(name) = &decorator.expression else { - return Err(Decline::new("only a plain-name decorator is lowered yet")); - }; - decorators.push(name.id.to_string()); - } + for decorator in function.decorator_list.iter().filter(|_| !nested) { + match function_modifier(db, model, decorator)? { + Modifier::Apply(name) => decorators.push(name), + Modifier::Erased | Modifier::DataClass => {} + } + } + // a method's first parameter is forced to the receiver, because that is what python + // puts in slot zero — and `staticmethod` and `classmethod` are exactly the two that + // say it is not. the method table entry says which, so the decorator is honoured by + // the emitted type rather than applied to it, and comes off the list here + let binding = method_binding(function, receiver, &mut decorators)?; + if receiver.is_none() { + decorator_stays_unread(unit.read, function.name.as_str(), &decorators)?; + } + // a class method's slot zero holds the *class*: an ordinary object, and pointedly + // not an instance of the layout, so nothing reads a field off it. a static method + // has no slot zero at all, and its first written parameter keeps its own type + let class_object = RType::OBJECT; + // what the `def` was *written* as a method of, which a zero-argument `super()` is + // asked about — it has its own account of why neither of these fills slot zero with + // a receiver, and the effective one no longer says which class the method is on + let declared_receiver = receiver; + let receiver = match binding { + Binding::Instance => receiver, + Binding::Static => None, + Binding::Class => Some(Receiver::Explicit(&class_object)), + }; // a generator and a coroutine do not run their body when called: they allocate a // state object and hand it back. the body becomes a method of that object @@ -2764,10 +3729,21 @@ fn lower_function_with_receiver( computed_defaults, } = signature(db, env, model, function, layouts, receiver, arrays)?; + // a boundary that hands the call on takes the twin off the interpreted class, and + // for a class method that is already *bound* — to the interpreted class, not to the + // one in slot zero. handing it the class as well would give the body two of them + if binding == Binding::Class && !computed_defaults.is_empty() { + return Err(Decline::new( + "a `classmethod` whose default is not an immediate would reach a twin already bound to the interpreted class", + )); + } + // a nested function lives on a generated environment class, whose fields are // the captures. it has to exist before the body is lowered, because the `def` // statement allocates it - let locals_here = local_representations(db, env, model, &function.body, layouts, unit.arrays); + let mut locals_here = + local_representations(db, env, model, &function.body, layouts, unit.arrays); + locals_here.retain(|(name, _)| !declared_global.contains(name)); let bound: HashSet = params .iter() .map(|(name, _)| name.clone()) @@ -2840,7 +3816,15 @@ fn lower_function_with_receiver( .collect(); let split = !cells_here.is_empty() && !bindings_here.is_empty(); - let frame_name = closures::environment_name(enclosing, &function.name); + // the *name* is qualified by the class the `def` was written in even where the + // frame has no receiver of that class, or a static `parse` and a module-level + // `parse` would ask for one environment class between them. the chain is not: + // `enclosing` is what says whether there is an outer frame to reach through, and + // neither of these has one + let frame_name = closures::environment_name( + enclosing.or_else(|| unit.owner.filter(|_| binding != Binding::Instance)), + &function.name, + ); let frame_owned: HashSet = if split { owned.difference(&bindings_here).cloned().collect() } else { @@ -3088,7 +4072,7 @@ fn lower_function_with_receiver( (None, _) => None, }; - let zero_super = zero_argument_super(function, receiver); + let zero_super = zero_argument_super(function, declared_receiver); let mut lowering = Lowering { arrays: unit.arrays, @@ -3099,7 +4083,9 @@ fn lower_function_with_receiver( model, builder, locals, + globals: declared_global, native_callees, + decorated: unit.decorated, layouts, methods, signatures, @@ -3146,7 +4132,8 @@ fn lower_function_with_receiver( lowering.builder.terminate(Terminator::Return(value)); } - let lowered = lowering.builder.finish(); + let mut lowered = lowering.builder.finish(); + lowered.binding = binding; // the environment's methods are the nested bodies, lowered with the environment // as the receiver — so a captured read is a field read like any other let environments = match environment { @@ -3240,6 +4227,11 @@ struct Unit<'a> { db: &'a dyn ty_python_semantic::Db, model: &'a SemanticModel<'a>, native_callees: &'a HashSet, + /// the module-level functions whose name a decorator rebinds, so a call through + /// that name has to resolve it rather than reach the native entry + decorated: &'a HashSet, + /// every name the module reads anywhere — see [`names_read`] + read: &'a BTreeSet<&'a str>, /// the module body, so a class can be asked about a base's base — which is what /// says whether its own fields sit inside a base's instance or past one suite: &'a [Stmt], @@ -3283,6 +4275,16 @@ struct Captured { #[derive(Clone)] enum Place { Register(RegisterId), + /// a name this frame declares `global`: it lives in the module namespace, and + /// neither half of it is a register + /// + /// the declaration is not a hint the write can ignore. python's binding is the + /// module's, so a write is visible at once to every other reader — and *this* + /// frame's own later reads have to come back out of the namespace too, or the + /// two halves stop agreeing with each other rather than with the module + Global { + name: String, + }, /// a field of a receiver register: a capture neither frame writes, copied in /// where the `def` runs Field { @@ -4258,6 +5260,44 @@ fn buffer_safe(body: &[Stmt], name: &str, arrays: &ArrayEditions) -> bool { mentions.iter().all(|mention| safe.contains(mention)) } +/// every name any frame in this module declares `global`, at any depth +/// +/// a `global` declaration is written in order to rebind: the name stops holding what +/// the module bound at import, and a call or a construction through it has to find +/// what is really there. that is exactly what a decorator does to a name, so the two +/// share a set — see `decorated` in [`build_module`]. +/// +/// a declaration with nothing assigned under it would be harmless, and is also +/// pointless, so this does not try to tell the two apart: naming one name too many +/// costs the direct call and nothing else, where naming one too few is a call that +/// reaches a definition the namespace no longer holds +fn declared_global_anywhere(body: &[Stmt]) -> HashSet { + let mut out = declared_globals(body); + for stmt in walk(body) { + match stmt { + Stmt::FunctionDef(node) => out.extend(declared_global_anywhere(&node.body)), + Stmt::ClassDef(node) => out.extend(declared_global_anywhere(&node.body)), + _ => {} + } + } + out +} + +/// the names a frame declares `global` +/// +/// [`walk`] stops at a nested `def` or `class`, which is what makes this per-scope: +/// a declaration inside one is that scope's, and python does not pass it outwards +fn declared_globals(body: &[Stmt]) -> HashSet { + walk(body) + .into_iter() + .filter_map(|stmt| match stmt { + Stmt::Global(node) => Some(node.names.iter().map(ast::Identifier::to_string)), + _ => None, + }) + .flatten() + .collect() +} + /// every statement in `body`, including nested ones fn walk(body: &[Stmt]) -> Vec<&Stmt> { let mut out = Vec::new(); @@ -4299,7 +5339,12 @@ struct Lowering<'a, 'db> { model: &'a SemanticModel<'db>, builder: FunctionBuilder, locals: HashMap, + /// the names this frame declares `global`, which live in the module namespace + /// rather than in any register of this frame + globals: HashSet, native_callees: &'a HashSet, + /// the module-level functions whose name a decorator rebinds + decorated: &'a HashSet, layouts: &'a Layouts, methods: &'a Methods, /// the signature of each module-level function, so a call coerces its arguments @@ -4490,6 +5535,21 @@ impl Lowering<'_, '_> { name: self.attribute_name(&attribute.attr), }); } + // a name in the module namespace *does* have an unbound state: + // it is not in the dict. a register does not, which is why the + // rest of this arm still declines + Expr::Name(name) + if matches!( + self.place(name.id.as_str()), + Some(Place::Global { .. }) + ) => + { + let status = self.builder.temp(RType::BIT); + self.builder.push(Op::DeleteGlobal { + dest: status, + name: name.id.to_string(), + }); + } _ => { return Err(Decline::new( "`del` on a plain name is not lowered yet — a register \ @@ -6304,6 +7364,13 @@ impl Lowering<'_, '_> { /// a register wins over a field: a generator's *parameters* are registers even /// where its locals are fields, and a closure's own parameters shadow a capture fn place(&self, name: &str) -> Option { + // asked first, and it has to be: a `global` declaration says this name is not + // this frame's to bind, so nothing else may answer for it + if self.globals.contains(name) { + return Some(Place::Global { + name: name.to_string(), + }); + } if let Some(&id) = self.locals.get(name) { return Some(Place::Register(id)); } @@ -6379,6 +7446,16 @@ impl Lowering<'_, '_> { let ty = self.register_type(*id)?; Ok((Value::Register(*id), ty)) } + // resolved out of the namespace exactly as an undeclared name is, which is + // what keeps a read after a write in the same frame seeing the write + Place::Global { name } => { + let dest = self.builder.temp(RType::OBJECT); + self.builder.push(Op::LoadGlobal { + dest, + name: name.clone(), + }); + Ok((Value::Register(dest), RType::OBJECT)) + } Place::Field { receiver, class, @@ -6443,6 +7520,17 @@ impl Lowering<'_, '_> { fn write_place(&mut self, place: &Place, value: Value, ty: &RType) -> Lowered<()> { match place { Place::Register(id) => self.store(*id, value, ty), + // the namespace holds objects, so an unboxed value is boxed on the way in + Place::Global { name } => { + let value = self.widen_to_object(value, ty); + let status = self.builder.temp(RType::BIT); + self.builder.push(Op::StoreGlobal { + dest: status, + name: name.clone(), + value, + }); + Ok(()) + } Place::Field { receiver, class, @@ -6783,11 +7871,21 @@ impl Lowering<'_, '_> { /// binding and unboxes the result back to an instance pointer. none of that /// says anything a direct allocation and a native `__init__` does not. /// + /// a **decorated** class is the one this does not answer for. the decorator is + /// applied to the namespace entry, and a construction is written against that name + /// — so allocating the emitted layout skips the decorator entirely, and a decorator + /// that returns another class had every construction in the module building the + /// wrong object. that one has to go out through the namespace and find what is + /// really there. + /// /// only a plain positional call: a default or a keyword needs the binding the /// signature describes, and falling back to the interpreted path for those is /// correct — just slower fn construct(&mut self, name: &str, node: &ast::ExprCall) -> Lowered> { - if !self.layouts.contains_key(name) || !node.arguments.keywords.is_empty() { + if !self.layouts.contains_key(name) + || self.decorated.contains(name) + || !node.arguments.keywords.is_empty() + { return Ok(None); } let Some(signature) = self.signatures.get(&qualify(Some(name), "__init__")) else { @@ -7277,17 +8375,27 @@ impl Lowering<'_, '_> { // is applied last — the same order the `def` statement itself applies them let mut made = Value::Register(closure); for decorator in node.decorator_list.iter().rev() { - let Expr::Name(name) = &decorator.expression else { - return Err(Decline::new( - "only a plain-name decorator on a nested function is lowered yet", - )); - }; let wrapped = self.builder.temp(RType::OBJECT); - self.builder.push(Op::CallPython { - dest: wrapped, - callee: name.id.to_string(), - args: vec![made], - }); + // the decorator expression is evaluated *here*, where the `def` stands, in + // this frame — which is what python does and what makes an arbitrary + // expression safe to take: `@functools.wraps(func)` reads `func` out of + // this frame's own registers, at the moment the closure is made + if let Expr::Name(name) = &decorator.expression + && !self.binds(name.id.as_str()) + { + self.builder.push(Op::CallPython { + dest: wrapped, + callee: name.id.to_string(), + args: vec![made], + }); + } else { + let callee = self.callable(&decorator.expression)?; + self.builder.push(Op::CallValue { + dest: wrapped, + callee, + args: vec![made], + }); + } made = Value::Register(wrapped); } let dest_ty = self.register_type(dest)?; @@ -7609,11 +8717,12 @@ impl Lowering<'_, '_> { Expr::Name(node) => { let name = node.id.as_str(); match self.place(name) { - Some(place) => self.read_place(&place), - // a name this frame does not have is a global, resolved the way - // `LOAD_GLOBAL` resolves it. the result is an `object`, so the - // checker's type for the expression decides any narrowing - None => { + Some(Place::Global { .. }) | None => { + // a name this frame does not have is a global, resolved the way + // `LOAD_GLOBAL` resolves it — and a name it *declares* `global` + // is the same read, which is what makes the declaration mean + // anything. the result is an `object`, so the checker's type for + // the expression decides any narrowing let dest = self.builder.temp(RType::OBJECT); self.builder.push(Op::LoadGlobal { dest, @@ -7621,6 +8730,7 @@ impl Lowering<'_, '_> { }); self.narrow_call_result(dest, expr) } + Some(place) => self.read_place(&place), } } // a `yield` is a field write and a return. the code after it becomes a new @@ -9112,11 +10222,13 @@ impl Lowering<'_, '_> { { return self.call_unpacked(node); } - // keywords the compiler cannot bind here — a method, or a name the unit does - // not own — are bound by python, from a tuple and a dict + // keywords the compiler cannot bind here — a method, a name the unit does not + // own, or one a decorator rebound to a signature this unit never saw — are + // bound by python, from a tuple and a dict if !node.arguments.keywords.is_empty() && !matches!(node.func.as_ref(), Expr::Name(name) - if self.native_callees.contains(name.id.as_str())) + if self.native_callees.contains(name.id.as_str()) + && !self.decorated.contains(name.id.as_str())) { return self.call_unpacked(node); } @@ -9223,8 +10335,16 @@ impl Lowering<'_, '_> { // a name the unit does not own is resolved and called the way the // interpreter would, with everything boxed on both sides. a call the native - // entry cannot take goes the same way, and reaches the deferring boundary - if !self.native_callees.contains(name) || self.defers_call(name, node) { + // entry cannot take goes the same way, and reaches the deferring boundary. + // + // a *decorated* one goes the same way for a different reason: the name holds + // what the decorator returned, and the native entry is what it was handed. + // reaching it directly would skip the decorator entirely — which is a wrong + // answer rather than a missed optimization + if !self.native_callees.contains(name) + || self.decorated.contains(name) + || self.defers_call(name, node) + { let mut args = Vec::with_capacity(node.arguments.args.len()); for argument in &node.arguments.args { let (value, ty) = self.expression(argument)?; diff --git a/crates/by_irbuild/src/single_file.rs b/crates/by_irbuild/src/single_file.rs index 3a9f705e50..9e1042e5dc 100644 --- a/crates/by_irbuild/src/single_file.rs +++ b/crates/by_irbuild/src/single_file.rs @@ -64,7 +64,7 @@ fn make_db(source: &str, language: Language) -> (TestDb, File) { /// lower source into a module named `module_name` pub fn module_from_source( source: &str, - module_name: &str, + module_name: impl Into, language: Language, ) -> by_ir::function::ModuleIr { with_source_in(source, language, |db, env, model, suite| { diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index a14b7fee5c..b74ae14c0b 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -22,6 +22,15 @@ fn has_op(function: &by_ir::function::Function, predicate: impl Fn(&Op) -> bool) .any(predicate) } +/// each decorator as it was written, which is what a test about *which* decorators +/// travel with a definition wants to read +fn dotted(decorators: &[by_ir::function::Decorator]) -> Vec { + decorators + .iter() + .map(by_ir::function::Decorator::dotted) + .collect() +} + /// lower `source` and render the module's IR, failing if it does not verify fn ir(source: &str) -> String { with_source(source, |db, env, model, suite| { @@ -1038,6 +1047,34 @@ class Tagged: ); } +#[test] +fn a_decorated_class_with_a_class_level_constant_is_lowered() { + // the constant is copied off the body the interpreted `class` statement wrote, which + // is captured while that statement runs and so predates every decorator. it used to + // be read back off the finished definition, and a decorator that makes something of + // what the body wrote leaves that definition saying something else — `@dataclass` + // deletes the `field(init=False)` a body wrote. so this whole shape declined, which + // over the corpus was almost every decorated class there was + let reasons = declines( + "\ +def tagger(cls): + return cls + + +@tagger +class Tagged: + KIND: str = \"class-level\" + + def read(self) -> str: + return \"read\" +", + ); + assert!( + !reasons.iter().any(|(name, _)| name == "Tagged"), + "{reasons:?}" + ); +} + #[test] fn a_private_class_constant_takes_its_mangled_name() { // python binds `__params` written in `class Function` as `_Function__params`, so @@ -1256,17 +1293,17 @@ class Held: } #[test] -fn a_class_level_constant_under_a_class_keyword_declines() { - // a constant keeps its class off the metaclass construction, because it is settled - // after the metaclass has already decided what the class defines. a class keyword - // leaves nowhere else to go — a type spec has nowhere to put the keyword — so what - // would answer is the interpreted definition, and that is only there while the - // module still holds the name. `ast` pops `Num` straight out of its own globals, so - // leaving this to the runtime turns the import into a `NameError`. +fn a_class_level_constant_under_a_class_keyword_is_carried_not_declined() { + // a constant used to keep its class off the metaclass construction, on the reasoning + // that it is settled after the metaclass has decided what the class defines. it is + // not: it goes into the namespace with the methods, and the class is asked afterwards + // whether it kept the value. so the class lowers, carrying the constant. // - // `Plain` is the boundary: the same keyword with no constant still lowers - let reasons = declines( - "\ + // a decorated method is the boundary, and the one thing this gate still turns down — + // what a decorator produces is only knowable by running it on a finished class + assert_eq!( + class_constants( + "\ from abc import ABCMeta @@ -1275,23 +1312,61 @@ class Tagged(metaclass=ABCMeta): def label(self) -> str: return \"tagged\" - - -class Plain(metaclass=ABCMeta): - def label(self) -> str: - return \"plain\" ", + "Tagged" + ), + vec!["TAG".to_string()] ); assert_eq!( - reasons, + declines( + "\ +from abc import ABCMeta + + +class Decorated(metaclass=ABCMeta): + @staticmethod + def label() -> str: + return \"decorated\" +" + ), vec![( - "Tagged".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" + "Decorated".to_string(), + "a decorated method on a class built through its metaclass is not lowered yet" .to_string() )] ); } +#[test] +fn a_class_level_constant_beside_a_base_of_ours_keeps_that_base_emitted() { + // the shape the stdlib is made of: no keyword at all, a base this module emits + // standing beside one from outside, and a constant. a spec cannot work that base list + // out, so the metaclass is what builds it — and the constant rides into the namespace + // it is handed rather than closing it. + // + // the base staying emitted is the point, and it is what the decline used to cost: + // `Reader` declining took `Codec` with it, because the interpreted `Reader` extends + // the *twin's* `Codec` and `issubclass(m.Reader, m.Codec)` would answer False against + // a twin that says True. neither declines now + const SOURCE: &str = "\ +import codecs + + +class Codec(codecs.Codec): + def label(self) -> str: + return \"codec\" + + +class Reader(Codec, codecs.StreamReader): + tag = 1 + + def kind(self) -> str: + return \"reader\" +"; + assert_eq!(declines(SOURCE), vec![]); + assert_eq!(class_constants(SOURCE, "Reader"), vec!["tag".to_string()]); +} + #[test] fn a_base_this_module_lays_out_declines_beside_one_it_does_not() { // a class holding both kinds of base takes its whole layout from outside, so a base @@ -1339,6 +1414,187 @@ class OnFieldless(Fieldless, codecs.Codec): ); } +#[test] +fn a_base_written_as_an_alias_is_the_class_it_was_bound_to() { + // the alias is not a base out of this module: it stands for a class this module + // writes, and the emitted type is what the name will hold. taking it as external + // built the class on the interpreted definition instead — the alias is carried over + // to the emitted type only once every class has been built — so `isinstance` said + // `False` where the interpreter says `True` + const SOURCE: &str = "\ +class Root: + def root(self) -> str: + return \"root\" + + +Alias = Root + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![ + ("Root".to_string(), None), + ( + "Over".to_string(), + Some(ClassBase::InModule("Root".to_string())) + ) + ] + ); +} + +#[test] +fn an_alias_does_not_hide_a_base_this_module_lays_out() { + // the same refusal as the direct spelling, which is the point: every question the + // base list is asked is asked of the *name*, so an alias that stood for itself + // walked straight past this gate and compiled the one shape it exists to refuse + let reasons = declines( + "\ +import codecs + + +class Laid: + def __init__(self, n: int) -> None: + self.n = n + + +Alias = Laid + + +class OnLaid(Alias, codecs.Codec): + pass +", + ); + assert_eq!( + reasons, + vec![ + ( + "OnLaid".to_string(), + "a base this module lays out cannot stand beside one it does not".to_string() + ), + ( + "Laid".to_string(), + "`OnLaid` declined, so it extends the interpreted definition rather than this type" + .to_string() + ) + ] + ); +} + +#[test] +fn a_name_the_module_binds_twice_declines_rather_than_pick_a_class() { + // `Over` was built on `Root` and the name holds `Other` by the time the module body + // ends, so neither class is the answer: the one the class statement saw is gone, and + // the one the emitted module would look up is not what it extends + let reasons = declines( + "\ +class Root: + def root(self) -> str: + return \"root\" + + +class Other: + def other(self) -> str: + return \"other\" + + +Alias = Root +Alias = Other + + +class Over(Alias): + def side(self) -> str: + return \"over\" +", + ); + assert_eq!( + reasons, + vec![( + "Over".to_string(), + "a base the module binds more than once stands for the class bound last, not the one it was built on" + .to_string() + )] + ); +} + +#[test] +fn an_alias_chain_that_leaves_the_module_is_left_where_it_was_written() { + // the emitted module looks the base up by name, and both names hold the same object + // at import — so following one is no gain, and it would trade a name this body binds + // once for one it may bind again. only a chain that ends at a class of *ours* moves + const SOURCE: &str = "\ +from codecs import Codec + +Alias = Codec + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Over".to_string(), + Some(ClassBase::External(vec!["Alias".to_string()])) + )] + ); +} + +#[test] +fn a_name_bound_twice_to_nothing_of_ours_still_stands_for_itself() { + // the boundary: a name the module rebinds is a hazard only where a class of this + // module's is behind it. two imported names leave the base exactly what it was + const SOURCE: &str = "\ +import codecs + +Alias = codecs.Codec +Alias = codecs.StreamWriter + + +class Over(Alias): + def side(self) -> str: + return \"over\" +"; + assert_eq!(declines(SOURCE), Vec::new()); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Over".to_string(), + Some(ClassBase::External(vec!["Alias".to_string()])) + )] + ); +} + #[test] fn a_class_with_fields_of_its_own_declines_beside_a_base_this_module_emits() { // the other half: the base of ours lays nothing out, but *this* class does. only a @@ -1639,14 +1895,205 @@ class Beside(Rooted): } #[test] -fn an_annotated_class_attribute_under_a_class_keyword_declines_with_the_rest() { - // an annotated assignment is a class-level constant, so it reaches the same gate a - // plain one does. this is what making the annotation a binding costs: a class the - // compiler used to build silently without the attribute now refuses to build at all. - // over the stdlib that cost was nothing — every class this reason reaches was - // already declining for another - let reasons = declines( +fn a_subclass_that_appends_nothing_past_a_base_declares_nothing() { + // the fields are what makes the difference: `Held` above appends storage past a + // `Wrapper` instance and has no construction, while a class that adds *no* field of + // its own appends nothing at all. what such a class keeps is what `Wrapper` already + // keeps, at the offsets `Wrapper` laid them out and through the descriptors `Wrapper` + // published — so it is built the way any other class with no storage of its own is + // + // `Restating` is the same class written the other way round: assigning an attribute + // the base already stores adds nothing either, and the write lands on the base's + // field through the base's own setter + let (declined, layouts) = with_source( "\ +class Wrapper(OSError): + def __init__(self, code: int) -> None: + self.code = code + + +class Plain(Wrapper): + pass + + +class Tagged(Wrapper): + TAG = 1 + + +class Restating(Wrapper): + def __init__(self, code: int) -> None: + self.code = code + 1 +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + let declined: Vec<(String, String)> = module + .declined + .iter() + .map(|declined| (declined.name.clone(), declined.reason.clone())) + .collect(); + let layouts: Vec<(String, Vec)> = module + .classes + .iter() + .map(|class| { + ( + class.name.clone(), + class + .fields + .iter() + .map(|field| field.name.clone()) + .collect(), + ) + }) + .collect(); + (declined, layouts) + }, + ); + assert_eq!(declined, Vec::new()); + assert_eq!( + layouts, + vec![ + ("Wrapper".to_string(), vec!["code".to_string()]), + ("Plain".to_string(), Vec::new()), + ("Tagged".to_string(), Vec::new()), + ("Restating".to_string(), Vec::new()), + ] + ); +} + +#[test] +fn a_subclass_with_no_storage_is_rebuilt_on_a_base_that_declines_later() { + // a base is settled as one of ours while the layouts settle, and only the body being + // lowered can turn it down after that. a class with no storage of its own does not + // need it to have stayed one: what stands under the name at import is a class either + // way, and building on the *name* is what every class over a base out of this module + // already does — so it takes that construction rather than cascading behind the base. + // + // `Storing` is the boundary: a class with a field declares a size of its own, and a + // class over a base out of this module declares none — the base allocates. so the + // storage would have nowhere to go, and it cascades behind the base instead + const SOURCE: &str = "\ +class Base: + def __new__(cls) -> \"Base\": + return object.__new__(cls) + + def label(self) -> str: + return \"base\" + + +class Below(Base): + def side(self) -> str: + return \"below\" + + +class Storing(Base): + def __init__(self, extra: int) -> None: + self.extra = extra +"; + assert_eq!( + declines(SOURCE), + vec![ + ( + "Base".to_string(), + "`__new__` fills a type slot with no adapter yet".to_string() + ), + ( + "Storing".to_string(), + "`Base` declined, so it is not a base to build on".to_string() + ) + ] + ); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Below".to_string(), + Some(ClassBase::External(vec!["Base".to_string()])) + )] + ); +} + +#[test] +fn a_class_a_subclass_stores_inside_is_not_rebuilt_on_its_declining_base() { + // rebuilding `Middle` on a name would move the layout of everything under it outside + // the module, and `Storing`'s field would go from sitting inside a `Middle` instance + // to sitting past one — a construction that has no answer, and one that refuses the + // *whole module* at import rather than the class. `urllib.request` lost all nineteen + // of its compiled functions that way. + // + // `Aside` is the boundary: nothing stores anything under it, so it is rebuilt + const SOURCE: &str = "\ +class Base: + def __new__(cls) -> \"Base\": + return object.__new__(cls) + + def label(self) -> str: + return \"base\" + + +class Middle(Base): + def side(self) -> str: + return \"middle\" + + +class Storing(Middle): + def __init__(self) -> None: + self.extra = 1 + + +class Aside(Base): + def side(self) -> str: + return \"aside\" +"; + assert_eq!( + declines(SOURCE), + vec![ + ( + "Base".to_string(), + "`__new__` fills a type slot with no adapter yet".to_string() + ), + ( + "Middle".to_string(), + "`Base` declined, so it is not a base to build on".to_string() + ), + ( + "Storing".to_string(), + "`Middle` declined, so it is not a base to build on".to_string() + ) + ] + ); + let bases = with_source(SOURCE, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .classes + .iter() + .map(|class| (class.name.clone(), class.base.clone())) + .collect::>() + }); + assert_eq!( + bases, + vec![( + "Aside".to_string(), + Some(ClassBase::External(vec!["Base".to_string()])) + )] + ); +} + +#[test] +fn an_annotated_class_attribute_under_a_class_keyword_is_carried_with_the_rest() { + // an annotated assignment is a class-level constant, so it takes the same route a + // plain one does — into the namespace the metaclass is handed. it used to reach the + // same gate instead, and a class the compiler had been building silently without the + // attribute went from missing the attribute to refusing to build at all + assert_eq!( + class_constants( + "\ from abc import ABCMeta @@ -1656,24 +2103,23 @@ class Tagged(metaclass=ABCMeta): def label(self) -> str: return \"tagged\" ", - ); - assert_eq!( - reasons, - vec![( - "Tagged".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" - .to_string() - )] + "Tagged" + ), + vec!["TAG".to_string()] ); } #[test] -fn a_subclass_of_a_class_the_metaclass_gates_turn_down_builds_on_the_interpreted_base() { - // both gates are asked while the layouts settle, so a class either of them turns - // down leaves the layout set — and its subclass then takes the external base every - // other declining class's subclass takes rather than being laid out on a base - // nothing emits. asked while the *body* was lowered instead, the base stayed in the - // set and both subclasses cascaded behind it +fn a_subclass_of_a_class_the_metaclass_gate_turns_down_builds_on_the_interpreted_base() { + // the gate is asked while the layouts settle, so a class it turns down leaves the + // layout set — and its subclass then takes the external base every other declining + // class's subclass takes rather than being laid out on a base nothing emits. asked + // while the *body* was lowered instead, the base stayed in the set and the subclass + // cascaded behind it. + // + // `Constant` is the boundary in the other direction: a class-level constant no longer + // turns a class down, so that half stays in the layout set and its subclass is laid + // out on it — an `InModule` base against an `External` one const SOURCE: &str = "\ from abc import ABCMeta @@ -1702,21 +2148,15 @@ class BelowConstant(Constant): "; assert_eq!( declines(SOURCE), - vec![ - ( - "Decorated".to_string(), - "a decorated method on a class built through its metaclass is not lowered yet" - .to_string() - ), - ( - "Constant".to_string(), - "a class-level constant on a class built through its metaclass is not lowered yet" - .to_string() - ) - ] + vec![( + "Decorated".to_string(), + "a decorated method on a class built through its metaclass is not lowered yet" + .to_string() + )] ); - // the base each subclass gets is the point: an `InModule` one would name a type - // this module never emits + // the base each subclass gets is the point: an `InModule` one below `Decorated` would + // name a type this module never emits, and an `External` one below `Constant` would + // give up a layout the module does have let bases = with_source(SOURCE, |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); module @@ -1732,9 +2172,14 @@ class BelowConstant(Constant): "BelowDecorated".to_string(), Some(ClassBase::External(vec!["Decorated".to_string()])) ), + // a keyword-only class header has no bases at all + ( + "Constant".to_string(), + Some(ClassBase::External(Vec::new())) + ), ( "BelowConstant".to_string(), - Some(ClassBase::External(vec!["Constant".to_string()])) + Some(ClassBase::InModule("Constant".to_string())) ) ] ); @@ -2060,13 +2505,11 @@ def plain(p: Plain) -> int: #[test] fn a_final_receiver_keeps_its_direct_call() { - // AB3 and AB4 traded the direct call away for any class that is decorated or in - // an inheritance chain. `@final` is about the *place*: nothing can subclass it, - // so there is no override for the protocol to find + // AB3 and AB4 traded the direct call away for any class in an inheritance chain. + // `final` is about the *place*: nothing can subclass it, so there is no override + // for the protocol to find with_source( "\ -from typing import final - data class Open: n: int @@ -2076,8 +2519,7 @@ data class Open: data class Derived(Open): extra: int -@final -data class Fixed(Open): +final data class Fixed(Open): label: str def tripled(self) -> int: @@ -2162,23 +2604,31 @@ def through(s: Shape) -> str: } #[test] -fn a_decorated_class_gives_up_its_direct_call() { +fn a_decorated_class_gives_up_its_direct_call_and_its_direct_construction() { // a decorated class is a mutable heap type, and python can rebind a method on - // one — a direct call would not see the rebinding + // one — a direct call would not see the rebinding. + // + // the *construction* goes the same way, and for a sharper reason: the decorator + // replaces what the module namespace binds, so `Loud(...)` names whatever it + // returned. allocating the emitted layout instead skipped the decorator outright, + // and a decorator returning another class had every construction in the module + // building the wrong object with no diagnostic at all with_source( "\ def tagged(cls: type) -> type: return cls @tagged -data class Loud: - n: int +class Loud: + def __init__(self, n: int) -> None: + self.n = n def doubled(self) -> int: return self.n * 2 -data class Quiet: - n: int +class Quiet: + def __init__(self, n: int) -> None: + self.n = n def doubled(self) -> int: return self.n * 2 @@ -2188,6 +2638,12 @@ def loud(x: Loud) -> int: def quiet(x: Quiet) -> int: return x.doubled() + +def build_loud(n: int) -> int: + return Loud(n).doubled() + +def build_quiet(n: int) -> int: + return Quiet(n).doubled() ", |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); @@ -2211,17 +2667,65 @@ def quiet(x: Quiet) -> int: "{}", ops("loud") ); + // and the undecorated one is allocated where it stands, while the + // decorated one is resolved through the namespace the decorator wrote to + assert!( + ops("build_quiet").contains("new Quiet"), + "{}", + ops("build_quiet") + ); + assert!( + !ops("build_loud").contains("new Loud"), + "{}", + ops("build_loud") + ); + assert!( + ops("build_loud").contains("pycall Loud"), + "{}", + ops("build_loud") + ); // and its decorators travel with the class let loud = module .classes .iter() .find(|class| class.name == "Loud") .expect("Loud is emitted"); - assert_eq!(loud.decorators, ["tagged"]); + assert_eq!(dotted(&loud.decorators), ["tagged"]); }, ); } +#[test] +fn a_class_modifier_is_not_a_decorator() { + // `sealed`, `abstract`, `open` and `export` reach the ast as decorators with no + // `@`, and the transpiler erases them — so the interpreted twin has no such name + // and looking one up at module init raised `NameError` and took the whole + // extension down with it + for modifier in ["sealed", "abstract", "open", "export"] { + let source = format!( + "\ +{modifier} data class Shape: + n: int +" + ); + let decorators = with_source(&source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{modifier}: {:?}", + module.declined + ); + module + .classes + .iter() + .find(|class| class.name == "Shape") + .map(|class| dotted(&class.decorators)) + .unwrap_or_else(|| panic!("{modifier}: Shape is emitted")) + }); + assert!(decorators.is_empty(), "{modifier}: {decorators:?}"); + } +} + #[test] fn a_class_with_a_hand_written_dunder_is_declined() { // `__init__` is generated from the fields, so a hand-written one would @@ -2246,82 +2750,518 @@ data class Point: } #[test] -fn len_is_an_intrinsic_not_a_call() { - let ir = ir("def f(s: str) -> int:\n return len(s)\n"); - assert!(ir.contains("= len "), "{ir}"); - assert!(!ir.contains("call len"), "{ir}"); +fn len_is_an_intrinsic_not_a_call() { + let ir = ir("def f(s: str) -> int:\n return len(s)\n"); + assert!(ir.contains("= len "), "{ir}"); + assert!(!ir.contains("call len"), "{ir}"); +} + +#[test] +fn a_module_defining_len_shadows_the_intrinsic() { + let ir = ir("\ +def len(x: int) -> int: + return x + +def f(a: int) -> int: + return len(a) +"); + assert!(ir.contains("call len(a)"), "{ir}"); +} + +#[test] +fn concatenating_two_strings_stays_a_string() { + let ir = ir("def f(a: str, b: str) -> str:\n return a + b\n"); + assert!(ir.contains("-> str"), "{ir}"); + assert!(ir.contains(" ++ "), "{ir}"); +} + +#[test] +fn an_async_function_is_declined() { + assert!(decline("def f(a: int) -> None:\n try:\n pass\n except* ValueError:\n pass\n") + .contains("`except*`")); +} + +#[test] +fn a_decorated_function_still_compiles() { + // the decorator is applied at module init to the installed native function, + // so the body is compiled either way + let source = "\ +def deco(f: object) -> object: + return f + +@deco +def f() -> None: + pass +"; + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let decorated = module + .functions + .iter() + .find(|function| function.name == "f") + .expect("f is compiled"); + assert_eq!(dotted(&decorated.decorators), ["deco"]); + }); +} + +#[test] +fn a_decorator_that_is_a_call_is_declined() { + // python calls `make(1)` where the `def` stands. module-level code is not compiled, + // so the only moment init has is the end of the module — by which time the + // interpreted twin has already made that call, and making it again would be a + // second one, in the wrong place + let source = "\ +def make(n: int) -> object: + return n + +@make(1) +def f() -> None: + pass +"; + let reason = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + module + .declined + .iter() + .find(|declined| declined.name == "f") + .map(|declined| declined.reason.clone()) + .unwrap_or_default() + }); + assert!(reason.contains("run it a second time"), "{reason}"); +} + +#[test] +fn a_decorator_written_as_a_path_keeps_its_segments() { + // every step of `functools.cache` is a read, so evaluating it at init means what it + // meant where the `def` stood — and the ir carries the chain rather than one name + let source = "\ +import functools + +@functools.cache +def f(n: int) -> int: + return n +"; + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let decorated = module + .functions + .iter() + .find(|function| function.name == "f") + .expect("f is compiled"); + assert_eq!( + decorated.decorators, + [by_ir::function::Decorator::Path { + root: "functools".to_string(), + attributes: vec!["cache".to_string()], + }] + ); + }); +} + +#[test] +fn a_decorator_rooted_in_the_class_body_is_declined() { + // a decorator is resolved out of the *module* namespace at init, and a class body is + // not that namespace. `@x.setter` is the shape this exists for, but that one also + // writes two `def`s of one name — so the names here are distinct, or the duplicate + // would answer first and this guard would never be reached + let source = "\ +class Box: + def wrap(fn: object) -> object: + return fn + + @wrap + def value(self) -> int: + return 1 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.classes.iter().all(|class| class.name != "Box"), + "Box must not be emitted" + ); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert!( + reasons + .iter() + .any(|reason| reason.contains("`wrap` is bound by the class body")), + "{reasons:?}" + ); +} + +#[test] +fn a_decorated_function_is_not_called_at_its_native_entry() { + // the module namespace holds what the decorator returned, and the native entry is + // what it was handed — so reaching the entry directly runs the *undecorated* + // function. `caller` answered 2 where the interpreted module answered 4 + let ir = ir("\ +def double(fn: object) -> object: + def inner(x: int) -> int: + return fn(x) * 2 + return inner + +@double +def f(x: int) -> int: + return x + 1 + +def caller(x: int) -> int: + return f(x) + +def plain(x: int) -> int: + return x + 1 + +def other(x: int) -> int: + return plain(x) +"); + assert!(ir.contains("pycall f("), "{ir}"); + assert!(!ir.contains("= call f("), "{ir}"); + // an undecorated sibling still reaches its native entry + assert!(ir.contains("= call plain("), "{ir}"); +} + +#[test] +fn a_modifier_is_translated_rather_than_looked_up() { + // a modifier reaches the ast as a decorator with no `@`, and there is no such name + // in the module namespace to look up: `static` compiled to + // `By_ApplyDecorator(dict, "make", "static")` and the extension then failed to + // import with `NameError: name 'static' is not defined`. + // + // the transpiler rewrites each of these to a python decorator, and this is the + // same mapping — so the compiled definition ends up wearing what the interpreted + // twin wears + for (modifier, expected) in [ + ("abstract", vec!["abstractmethod".to_string()]), + ("override", vec!["override".to_string()]), + ("export", Vec::new()), + ] { + let source = format!( + "\ +class Box: + {modifier} def make(self) -> int: + return 7 +" + ); + let decorators = with_source(&source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{modifier}: {:?}", + module.declined + ); + module + .all_functions() + .find(|function| function.name.ends_with("make")) + .map(|function| dotted(&function.decorators)) + .unwrap_or_else(|| panic!("{modifier}: make is compiled")) + }); + assert_eq!(decorators, expected, "{modifier}"); + } +} + +#[test] +fn a_method_that_is_not_bound_to_its_receiver_carries_its_convention() { + // a method's first parameter used to be forced to the receiver whatever the + // decorators said, and these two say slot zero holds something else — so + // `Box.make(3)` compiled with `3` bound to a `Box` and raised at its first call. + // + // the convention rides on the method table entry now, so slot zero holds what + // python puts there: nothing at all for a static method, whose first written + // parameter keeps its own representation, and the *class* for a class method — an + // ordinary object, pointedly not an instance of the layout, so nothing derives a + // field read from it. + // + // and the decorator comes off the list: it is honoured by the emitted type rather + // than applied to it, and applying it as well would wrap the descriptor twice + for (source, binding, params) in [ + ( + "class Box:\n static def make(x: int) -> int:\n return x\n", + by_ir::function::Binding::Static, + vec![("x", RType::INT)], + ), + ( + "class Box:\n @staticmethod\n def make(x: int) -> int:\n return x\n", + by_ir::function::Binding::Static, + vec![("x", RType::INT)], + ), + ( + "class Box:\n @classmethod\n def make(cls, x: int) -> int:\n return x\n", + by_ir::function::Binding::Class, + vec![("cls", RType::OBJECT), ("x", RType::INT)], + ), + ( + "class Box:\n def make(self, x: int) -> int:\n return x\n", + by_ir::function::Binding::Instance, + vec![ + ( + "self", + RType::Instance { + class: "Box".to_string(), + exact: false, + }, + ), + ("x", RType::INT), + ], + ), + ] { + with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{source}: {:?}", + module.declined + ); + let method = module + .all_functions() + .find(|function| function.name.ends_with("make")) + .unwrap_or_else(|| panic!("{source}: make is compiled")); + assert_eq!(method.binding, binding, "{source}"); + assert!(method.decorators.is_empty(), "{source}"); + let lowered: Vec<(&str, RType)> = method + .params() + .iter() + .map(|param| (param.name.as_deref().unwrap_or(""), param.ty.clone())) + .collect(); + let expected: Vec<(&str, RType)> = params + .iter() + .map(|(name, ty)| (*name, ty.clone())) + .collect(); + assert_eq!(lowered, expected, "{source}"); + }); + } } #[test] -fn a_module_defining_len_shadows_the_intrinsic() { - let ir = ir("\ -def len(x: int) -> int: - return x - -def f(a: int) -> int: - return len(a) -"); - assert!(ir.contains("call len(a)"), "{ir}"); +fn a_second_decorator_over_a_static_method_keeps_the_decline() { + // the runtime folds the remaining decorators onto the attribute it reads back off + // the finished type — and reading a static method back hands over the plain + // function it wraps, which would then be written back as an ordinary method. so + // the convention is only honoured natively where nothing else has to be applied + for source in [ + "class Box:\n @final\n @staticmethod\n def make() -> int:\n return 7\n", + "class Box:\n @staticmethod\n @final\n def make() -> int:\n return 7\n", + "class Box:\n @final\n @classmethod\n def make(cls) -> int:\n return 7\n", + ] { + let reasons = declines(source); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("a second decorator over")), + "{source}: {reasons:?}" + ); + } } #[test] -fn concatenating_two_strings_stays_a_string() { - let ir = ir("def f(a: str, b: str) -> str:\n return a + b\n"); - assert!(ir.contains("-> str"), "{ir}"); - assert!(ir.contains(" ++ "), "{ir}"); +fn a_convention_python_gives_a_method_itself_is_not_carried_twice() { + // python already makes each of these implicitly static or class, and an emitted + // generic class publishes a `__class_getitem__` of its own — so a table entry here + // would either duplicate the convention or collide with that entry + for source in [ + "class Box:\n @staticmethod\n def __new__(cls) -> object:\n return 7\n", + "class Box:\n @classmethod\n def __init_subclass__(cls) -> None:\n return None\n", + "class Box:\n @classmethod\n def __class_getitem__(cls, item: object) -> object:\n return item\n", + ] { + let reasons = declines(source); + assert!( + reasons + .iter() + .any(|(_, reason)| reason.contains("a convention of its own") + || reason.contains("fills a type slot")), + "{source}: {reasons:?}" + ); + } } #[test] -fn an_async_function_is_declined() { - assert!(decline("def f(a: int) -> None:\n try:\n pass\n except* ValueError:\n pass\n") - .contains("`except*`")); +fn a_global_a_frame_assigns_is_written_to_the_namespace_and_read_back_from_it() { + // a `global` declaration says where a name lives, and both halves of it have to + // agree: the write goes to the module namespace, and every read in the same frame + // comes back out of it. binding a register for either half is what made + // `mimetypes.init` set `inited` where nothing else could see it + for (source, function, writes, reads) in [ + ( + "seen = 0\n\ndef bump(n: int) -> int:\n global seen\n seen = n\n return seen\n", + "bump", + 1, + 1, + ), + // augmented assignment is a read and a write of the one place, so it is both + ( + "seen = 0\n\ndef bump(n: int) -> int:\n global seen\n seen += n\n return seen\n", + "bump", + 1, + 2, + ), + // a loop target is a binding like any other + ( + "seen = 0\n\ndef bump(ns: list[int]) -> int:\n global seen\n for seen in ns:\n pass\n return seen\n", + "bump", + 1, + 1, + ), + // a declaration with no assignment under it is redundant, and the name resolves + // where it already resolved + ( + "seen = 0\n\ndef read(n: int) -> int:\n global seen\n return seen + n\n", + "read", + 0, + 1, + ), + // the same name written by a frame that did *not* declare it is an ordinary + // local, and shadowing the global is what python does with it too + ( + "seen = 0\n\ndef shadow(n: int) -> int:\n seen = n\n return seen\n", + "shadow", + 0, + 0, + ), + ] { + let (stores, loads) = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.declined.is_empty(), + "{source}: {:?}", + module.declined + ); + let lowered = module + .functions + .iter() + .find(|candidate| candidate.name == function) + .unwrap_or_else(|| panic!("{function} was not emitted")); + let count = |wanted: fn(&Op) -> bool| { + lowered + .blocks + .iter() + .flat_map(|block| block.ops.iter()) + .filter(|op| wanted(op)) + .count() + }; + ( + count(|op| matches!(op, Op::StoreGlobal { name, .. } if name == "seen")), + count(|op| matches!(op, Op::LoadGlobal { name, .. } if name == "seen")), + ) + }); + assert_eq!((stores, loads), (writes, reads), "{source}"); + } } #[test] -fn a_decorated_function_still_compiles() { - // the decorator is applied at module init to the installed native function, - // so the body is compiled either way +fn a_global_a_nested_frame_declares_is_never_captured_from_the_frame_around_it() { + // the enclosing frame binds a local `seen` and the nested one declares `seen` + // global, so the two names are different places. the nested body only *reads* it, + // which is the case nothing else rules out: a body that wrote it would look like + // it owned the name anyway, so only the declaration says the enclosing local is + // the wrong place to capture let source = "\ -def deco(f: object) -> object: - return f +seen = 0 +tally = 0 -@deco -def f() -> None: - pass + +def outer(n: int) -> int: + seen = n + + def peek() -> int: + global seen + return seen + + return peek() + seen + + +def declared_out_here(n: int) -> int: + global tally + tally = n + + def look() -> int: + return tally + + return look() + tally "; - with_source(source, |db, env, model, suite| { + let (fields, loads) = with_source(source, |db, env, model, suite| { let module = crate::build_module(db, env, model, suite, "app", true); assert!(module.declined.is_empty(), "{:?}", module.declined); - let decorated = module - .functions + let nested: Vec<&by_ir::function::Function> = module + .classes .iter() - .find(|function| function.name == "f") - .expect("f is compiled"); - assert_eq!(decorated.decorators, vec!["deco".to_string()]); + .flat_map(|class| class.methods.iter()) + .filter(|candidate| candidate.name == "peek" || candidate.name == "look") + .collect(); + assert_eq!(nested.len(), 2, "both nested functions are emitted"); + ( + // neither name may become an environment field: there is nothing in the + // frame around either nested function for it to hold + module + .classes + .iter() + .flat_map(|class| class.fields.iter()) + .filter(|field| field.name == "seen" || field.name == "tally") + .count(), + nested + .iter() + .flat_map(|function| function.blocks.iter()) + .flat_map(|block| block.ops.iter()) + .filter(|op| { + matches!(op, Op::LoadGlobal { name, .. } if name == "seen" || name == "tally") + }) + .count(), + ) }); + assert_eq!((fields, loads), (0, 2)); } #[test] -fn a_computed_decorator_is_declined() { - // a call or an attribute would need its arguments evaluated at module init - let source = "\ -def make(n: int) -> object: - return n - -@make(1) -def f() -> None: - pass -"; - let reason = with_source(source, |db, env, model, suite| { - let module = crate::build_module(db, env, model, suite, "app", true); - module - .declined +fn a_static_method_that_suspends_declines() { + // a generator's state class is namespaced by the receiver's class, and neither of + // these has one — so two classes each with a static `values` would want a single + // state class between them + let reasons = + declines("class Box:\n @staticmethod\n def values() -> object:\n yield 1\n"); + assert!( + reasons .iter() - .find(|declined| declined.name == "f") - .map(|declined| declined.reason.clone()) - .unwrap_or_default() - }); - assert!(reason.contains("plain-name decorator"), "{reason}"); + .any(|(_, reason)| reason.contains("that suspends is not lowered yet")), + "{reasons:?}" + ); +} + +#[test] +fn a_static_method_and_a_function_of_one_name_get_environments_of_their_own() { + // a nested function lives on a generated environment class named after the frame + // that makes it, and a method's frame is namespaced by its class. a static method + // has no receiver to take that name from, so the name comes from the class the + // `def` was *written* in — otherwise these two ask for one class between them + with_source( + "\ +class Box: + @staticmethod + def add(n: int) -> int: + def inner(k: int) -> int: + return k + n + return inner(1) + + +def add(n: int) -> int: + def inner(k: int) -> int: + return k + n + 100 + return inner(1) +", + |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + let environments: Vec<&str> = module + .classes + .iter() + .map(|class| class.name.as_str()) + .filter(|name| name.ends_with("$env")) + .collect(); + assert_eq!(environments, vec!["Box$add$env", "add$env"]); + }, + ); } #[test] @@ -3063,7 +4003,7 @@ data class Point: .methods .iter() .find(|method| method.name == name) - .map(|method| method.decorators.clone()) + .map(|method| dotted(&method.decorators)) }; assert_eq!(decorators("total"), Some(vec!["property".to_string()])); assert_eq!(decorators("raw"), Some(vec!["doubling".to_string()])); @@ -4813,3 +5753,179 @@ class Held: ); assert_eq!(reasons, Vec::new()); } + +#[test] +fn a_method_defined_twice_in_a_class_body_is_declined() { + // two `def`s of one name bind whichever one ran, and they mangle to one C symbol — + // so a class with both emitted two `Box.value` entries and the module then failed to + // compile outright, which is worse than any wrong answer + let source = "\ +class Box: + def value(self) -> int: + return 1 + + def value(self) -> int: + return 2 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!( + module.classes.iter().all(|class| class.name != "Box"), + "Box must not be emitted" + ); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert!( + reasons + .iter() + .any(|reason| reason.contains("`value` is defined more than once")), + "{reasons:?}" + ); +} + +#[test] +fn a_module_level_function_defined_twice_is_declined() { + // the same in the module scope, which had nobody asking: three module-level `def _`s + // in `importlib/resources/_common.py` emitted one `by_m__` twice and the extension + // failed to build at all + let source = "\ +import functools + +@functools.cache +def _(n: int) -> int: + return n + +@functools.cache +def _(n: int) -> int: + return n + 1 +"; + let reasons = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.functions.is_empty(), "nothing may compile"); + module + .declined + .iter() + .map(|declined| declined.reason.clone()) + .collect::>() + }); + assert_eq!(reasons.len(), 2, "{reasons:?}"); + assert!( + reasons + .iter() + .all(|reason| reason.contains("`_` is defined more than once")), + "{reasons:?}" + ); +} + +/// the twin's source keeps only the decorators module init will not re-apply +/// +/// a decorator init applies is evaluated there, over the compiled definition — so +/// leaving it on the twin's `def` evaluates it a second time and doubles whatever it did +/// on the way. a class's comes out for the same reason, because init applies that one to +/// the namespace entry. a *method's* stays: the class construction reads what it wrote — +/// `ABCMeta` computes `__abstractmethods__` from the namespace the body left — so taking +/// it out changes the class the twin builds rather than only when the decorator ran +#[test] +fn only_the_decorators_init_applies_come_out_of_the_twin() { + let source = "\ +def mark(f: object) -> object: + return f + + +@mark +def counted() -> int: + return 1 + + +@mark +class Held: + @mark + def value(self) -> int: + return 2 +"; + let twin = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + crate::without_init_decorators(source, &module).expect("the twin parses") + }); + assert_eq!(twin.matches("@mark").count(), 1, "{twin}"); + assert!(twin.contains(" @mark\n def value"), "{twin}"); + // the blanking keeps every line where it was, so a traceback through the twin still + // quotes the right one + assert_eq!(twin.lines().count(), source.lines().count(), "{twin}"); +} + +/// a decorator init does not re-apply stays on the twin's definition +/// +/// `staticmethod` is the shape: the method table honours the binding itself, so init has +/// nothing to apply and the twin's own `def` is the only thing that can +#[test] +fn a_decorator_init_does_not_re_apply_stays_on_the_twin() { + let source = "\ +class Held: + @staticmethod + def value() -> int: + return 2 +"; + let twin = with_source(source, |db, env, model, suite| { + let module = crate::build_module(db, env, model, suite, "app", true); + assert!(module.declined.is_empty(), "{:?}", module.declined); + crate::without_init_decorators(source, &module).expect("the twin parses") + }); + assert!(twin.contains("@staticmethod"), "{twin}"); +} + +/// a decorated definition the module reads cannot have its decorator moved to init +#[test] +fn a_decorated_definition_the_module_reads_declines() { + let reasons = declines( + "\ +def mark(f: object) -> object: + return f + + +@mark +def counted() -> int: + return 1 + + +at_import = counted() +", + ); + assert!( + reasons + .iter() + .any(|(name, reason)| name == "counted" && reason.contains("this module reads")), + "{reasons:?}" + ); +} + +/// and neither can a decorated class the module reads +#[test] +fn a_decorated_class_the_module_reads_declines() { + let reasons = declines( + "\ +def mark(c: object) -> object: + return c + + +@mark +class Held: + def value(self) -> int: + return 1 + + +table = [Held] +", + ); + assert!( + reasons + .iter() + .any(|(name, reason)| name == "Held" && reason.contains("this module reads")), + "{reasons:?}" + ); +} diff --git a/crates/by_opt/src/coalesce.rs b/crates/by_opt/src/coalesce.rs index 6c0c2abf1a..01bf1718ad 100644 --- a/crates/by_opt/src/coalesce.rs +++ b/crates/by_opt/src/coalesce.rs @@ -112,7 +112,7 @@ mod tests { fn module(function: by_ir::function::Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], classes: Vec::new(), declined: Vec::new(), diff --git a/crates/by_opt/src/copy_propagation.rs b/crates/by_opt/src/copy_propagation.rs index 9b8928cdab..96cce8d981 100644 --- a/crates/by_opt/src/copy_propagation.rs +++ b/crates/by_opt/src/copy_propagation.rs @@ -272,6 +272,8 @@ fn retarget(op: &mut Op, new_dest: RegisterId) { | Op::CallPython { dest, .. } | Op::CallValue { dest, .. } | Op::LoadGlobal { dest, .. } + | Op::StoreGlobal { dest, .. } + | Op::DeleteGlobal { dest, .. } | Op::LoadClass { dest, .. } | Op::ImportModule { dest, .. } | Op::ImportFrom { dest, .. } @@ -341,7 +343,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/dead_registers.rs b/crates/by_opt/src/dead_registers.rs index 48d0500e26..9c271c5a9b 100644 --- a/crates/by_opt/src/dead_registers.rs +++ b/crates/by_opt/src/dead_registers.rs @@ -168,6 +168,13 @@ fn rewrite_op( | Op::ImportModule { dest, .. } => { rewrite_dest(dest); } + Op::StoreGlobal { dest, value, .. } => { + rewrite_dest(dest); + rewrite_value(value, remap); + } + Op::DeleteGlobal { dest, .. } => { + rewrite_dest(dest); + } Op::ImportFrom { dest, module, .. } => { rewrite_dest(dest); rewrite_value(module, remap); @@ -447,7 +454,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/fold.rs b/crates/by_opt/src/fold.rs index f68667e6ac..134f8545c1 100644 --- a/crates/by_opt/src/fold.rs +++ b/crates/by_opt/src/fold.rs @@ -520,7 +520,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/infallible.rs b/crates/by_opt/src/infallible.rs index eb4efbbbda..d14e237e7c 100644 --- a/crates/by_opt/src/infallible.rs +++ b/crates/by_opt/src/infallible.rs @@ -150,6 +150,8 @@ fn op_can_fail(module: &ModuleIr, function: &by_ir::function::Function, op: &Op) | Op::CallPython { .. } | Op::CallValue { .. } | Op::LoadGlobal { .. } + | Op::StoreGlobal { .. } + | Op::DeleteGlobal { .. } | Op::ImportModule { .. } | Op::ImportFrom { .. } | Op::NewInstance { .. } @@ -267,7 +269,7 @@ mod tests { fn module(functions: Vec) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions, declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/lib.rs b/crates/by_opt/src/lib.rs index b4e6e58574..6c826c1119 100644 --- a/crates/by_opt/src/lib.rs +++ b/crates/by_opt/src/lib.rs @@ -150,7 +150,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(index))); let mut module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), @@ -185,7 +185,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(out))); let mut module = ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![builder.finish()], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/refcount.rs b/crates/by_opt/src/refcount.rs index 4300c66cec..85b901dfa2 100644 --- a/crates/by_opt/src/refcount.rs +++ b/crates/by_opt/src/refcount.rs @@ -102,7 +102,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/str_append.rs b/crates/by_opt/src/str_append.rs index fa7b468894..77d1792b41 100644 --- a/crates/by_opt/src/str_append.rs +++ b/crates/by_opt/src/str_append.rs @@ -178,7 +178,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/str_item_compare.rs b/crates/by_opt/src/str_item_compare.rs index ff34080392..e6d2522f92 100644 --- a/crates/by_opt/src/str_item_compare.rs +++ b/crates/by_opt/src/str_item_compare.rs @@ -148,7 +148,7 @@ mod tests { fn module(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_opt/src/unswitch.rs b/crates/by_opt/src/unswitch.rs index 5c53df6efa..938fefa991 100644 --- a/crates/by_opt/src/unswitch.rs +++ b/crates/by_opt/src/unswitch.rs @@ -372,7 +372,7 @@ mod tests { fn module_with(function: Function) -> ModuleIr { ModuleIr { - name: "app".to_string(), + name: by_ir::ModuleName::new("app"), functions: vec![function], declined: Vec::new(), classes: Vec::new(), diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 4979a7e8bb..0b26ffa2e1 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -821,6 +821,29 @@ static inline char By_Truthy(PyObject *o) { #define By_TypeData(obj, cls) ((void *)(obj)) #endif +/* whether an emitted class may keep an instance dict beside its layout + * + * a managed dict lives in the pre-header, so it is the one form that leaves the struct, + * its base's prefix and every field offset alone — but walking and releasing it is + * `PyObject_VisitManagedDict` and `PyObject_ClearManagedDict`, which 3.13 published and + * nothing below it offers outside the internal headers. so a module holding such a class + * is left to its interpreted definitions on an older interpreter, decided at import + * rather than when the C is written */ +#if PY_VERSION_HEX >= 0x030D0000 +#define BY_HAS_MANAGED_DICT 1 +#define BY_MANAGED_DICT_FLAG Py_TPFLAGS_MANAGED_DICT +#define By_VisitManagedDict(obj, visit, arg) PyObject_VisitManagedDict((obj), (visit), (arg)) +#define By_ClearManagedDict(obj) PyObject_ClearManagedDict(obj) +#else +#define BY_HAS_MANAGED_DICT 0 +/* the flag is spelled in a static initializer, which is written whatever the interpreter + * — and below 3.11 there is no such flag to name at all. these three are never reached: + * the module falls back before any instance of such a class exists */ +#define BY_MANAGED_DICT_FLAG 0 +#define By_VisitManagedDict(obj, visit, arg) ((void)(obj), (void)(visit), (void)(arg)) +#define By_ClearManagedDict(obj) ((void)(obj)) +#endif + /* reading a local on a path that never assigned it. the phrasing is the running * python's, not the compiler's — it changed in 3.11 and a compiled module has to say * what the interpreter beside it would say */ @@ -892,6 +915,72 @@ static inline PyObject *By_LookupGlobalString(PyObject *dict, const char *name) return value; } +/* `root.a.b`: the root the way `LOAD_GLOBAL` resolves it, then a `getattr` each + * + * this is what a decorator expression written as a chain of attributes does, and all + * of what it does — every step is a read, which is why evaluating it at module init + * rather than where the `def` stood is faithful. a python identifier holds no `.`, so + * the path arrives as one string and is split back apart here */ +static inline PyObject *By_LookupDotted(PyObject *dict, const char *path) { + const char *dot = strchr(path, '.'); + PyObject *value; + if (dot == NULL) return By_LookupGlobalString(dict, path); + { + PyObject *key = By_InternedStr(path, (Py_ssize_t)(dot - path)); + if (key == NULL) return NULL; + value = By_LookupGlobal(dict, key); + Py_DECREF(key); + } + while (value != NULL && dot != NULL) { + const char *segment = dot + 1; + const char *next = strchr(segment, '.'); + Py_ssize_t length = next == NULL ? (Py_ssize_t)strlen(segment) + : (Py_ssize_t)(next - segment); + PyObject *attr = By_InternedStr(segment, length); + PyObject *got; + if (attr == NULL) { + Py_DECREF(value); + return NULL; + } + got = PyObject_GetAttr(value, attr); + Py_DECREF(attr); + Py_DECREF(value); + value = got; + dot = next; + } + return value; +} + +/* bind a name in the module namespace: an assignment under a `global` declaration + * + * this is the write `By_LookupGlobal` is the read of, and it has to reach the same + * dict. binding a register instead would keep the new value to the frame, where + * python's binding is the module's — every other reader sees it at once, the + * interpreted twin included, since that twin's `__globals__` *is* this dict. + * + * builtins are pointedly not consulted: python's `STORE_GLOBAL` binds in the module + * namespace whether or not the name already resolved to a builtin */ +static inline char By_StoreGlobal(PyObject *dict, PyObject *name, PyObject *value) { + if (dict == NULL || name == NULL || value == NULL) return 2; + return PyDict_SetItem(dict, name, value) < 0 ? 2 : 0; +} + +/* unbind a name in the module namespace: `del x` under a `global x` + * + * a dict raises `KeyError` for a key it does not hold and python raises `NameError` + * for a name it does not bind, so the one has to be translated into the other */ +static inline char By_DeleteGlobal(PyObject *dict, PyObject *name) { + if (dict == NULL || name == NULL) return 2; + if (PyDict_DelItem(dict, name) < 0) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + PyErr_Format(PyExc_NameError, "name '%U' is not defined", name); + } + return 2; + } + return 0; +} + /* whether a type spec can be built on this tuple of bases * * `PyType_FromSpecWithBases` gives the type it builds `type` as its own, so any base @@ -919,14 +1008,27 @@ static inline int By_SpecTakesBases(PyObject *bases) { * deallocation. * * a class statement works the shape out from every base at once, so where the offsets - * disagree with the layout base the interpreted definition is what answers */ -static inline int By_OffsetsHoldUp(PyTypeObject *type) { + * disagree with the layout base the interpreted definition is what answers. + * + * a spec that *asked* for a managed dict is the one exception, and the spec is passed in + * so that asking can be told from inheriting: python keeps a managed dict in a pre-header + * it allocates itself, so the room is there and the offset — the sentinel `-1` — is the + * answer that was wanted. without this a decorated class silently kept its interpreted + * definition while every compiled function went on reading that definition's instances as + * its own struct */ +static inline int By_OffsetsHoldUp(PyTypeObject *type, PyType_Spec *spec) { PyTypeObject *base = type->tp_base; if (base == NULL) { return 1; } - return type->tp_dictoffset == base->tp_dictoffset - && type->tp_weaklistoffset == base->tp_weaklistoffset; + if (type->tp_weaklistoffset != base->tp_weaklistoffset) { + return 0; + } + if (type->tp_dictoffset == base->tp_dictoffset) { + return 1; + } + return spec != NULL && (spec->flags & BY_MANAGED_DICT_FLAG) != 0 + && type->tp_dictoffset == -1; } /* the type for a class whose fields sit past a base's instance, or nothing at all @@ -987,7 +1089,74 @@ static inline PyObject *By_SpecClass(PyObject *module_dict, const char *name, PyErr_Clear(); return NULL; } - if (!By_OffsetsHoldUp((PyTypeObject *)cls)) { + if (!By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + +/* the type for a class appending storage past one *this module also appends to* + * + * `By_SpecClass` refuses a heap base outright, and a class this module writes is always + * one: the interpreted definition under that name is a `class` statement's type, whose + * `tp_dealloc` is `subtype_dealloc`. that refusal is the right answer for a base python + * built — `subtype_dealloc` picks the deallocator to chain to out of `Py_TYPE(self)`, + * finds this class's own, and calls it back until the stack runs out. + * + * a base *this module builds from a spec* is the one heap base that is not like that. + * its three slots are ones we emitted: each reads the base to chain to from the type + * that declared it, so the chain walks down to the outside base and stops. so the whole + * chain of appended storage can be built, innermost first, each spec standing on the + * finished type of the one below rather than on the interpreted definition. + * + * `base` is that finished type. what is checked here is that the interpreted definition + * agrees the two are related the way the emitted pair are: `base_name` is still the twin + * this module's `base` was built from — nothing of this module's own is installed yet — + * and the twin settled on exactly it, as its layout base and as its only base. anything + * else and the emitted type would answer a shape the source never wrote */ +static inline PyObject *By_SpecSubclass(PyObject *module_dict, const char *name, + PyType_Spec *spec, const char *base_name, + PyObject *base) { + PyObject *twin = By_LookupGlobalString(module_dict, name); + PyObject *twin_base; + PyObject *bases; + PyObject *cls; + int agrees; + if (twin == NULL) { + PyErr_Clear(); + return NULL; + } + twin_base = By_LookupGlobalString(module_dict, base_name); + if (twin_base == NULL) { + PyErr_Clear(); + Py_DECREF(twin); + return NULL; + } + agrees = PyType_Check(twin) && PyType_Check(twin_base) + && (PyObject *)((PyTypeObject *)twin)->tp_base == twin_base + && PyTuple_GET_SIZE(((PyTypeObject *)twin)->tp_bases) == 1 + && PyTuple_GET_ITEM(((PyTypeObject *)twin)->tp_bases, 0) == twin_base + && By_SpecTakesBases(((PyTypeObject *)twin)->tp_bases); + Py_DECREF(twin_base); + Py_DECREF(twin); + if (!agrees) { + return NULL; + } + bases = PyTuple_Pack(1, base); + if (bases == NULL) { + PyErr_Clear(); + return NULL; + } + cls = PyType_FromSpecWithBases(spec, bases); + Py_DECREF(bases); + if (cls == NULL) { + PyErr_Clear(); + return NULL; + } + /* the same last question `By_SpecClass` asks, and for the same reason: a spec adds + * neither a `__dict__` nor a weakref, so both offsets have to be the base's */ + if (!By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { Py_DECREF(cls); return NULL; } @@ -1126,6 +1295,120 @@ static inline int By_SetInNamespace(PyObject *ns, const char *key, PyObject *val return failed; } +/* what a value carried onto an emitted type becomes — defined with the rest of the twin + * machinery, and named here because a class namespace is written before that */ +static PyObject *By_TwinReplacement(PyObject *value, PyObject *const *twins, + PyObject *const *types, Py_ssize_t count); + +/* the class-level constants a class body wrote, and where their values come from + * + * the interpreted definition evaluated each of them once at class-definition time, and it + * is the only place the same object can come from — so the body that definition wrote is + * what they are read off, under the substitution every carried attribute takes. that body + * is captured while the fallback source runs and before any of the class's own decorators + * are handed it; `By_RunModuleBody` says why the finished class will not do. `twins` and + * `types` are the module's arrays and `classes` how many entries they hold; `body` is NULL + * for a class no interpreted `class` statement wrote, and then there is nothing to carry */ +typedef struct { + PyObject *body; + const char *const *names; + Py_ssize_t count; + PyObject *const *twins; + PyObject *const *types; + Py_ssize_t classes; +} By_ClassConstants; + +/* the value one of them takes, as a new reference + * + * NULL is "the body did not write that name", which is not a failure and leaves no + * exception set: a body under a conditional may not have written it */ +static inline PyObject *By_ConstantValue(const By_ClassConstants *constants, Py_ssize_t at) { + PyObject *value, *stands; + if (constants == NULL || constants->body == NULL) return NULL; + /* read out of the mapping rather than through a lookup on the class: a lookup runs + * the descriptor protocol, so a `__class_getitem__ = classmethod(f)` would come back + * as a method already bound to the interpreted class rather than as the classmethod + * the body wrote */ + value = PyDict_GetItemString(constants->body, constants->names[at]); + if (value == NULL) return NULL; + /* a value that only *reaches* a twin keeps what the body gave it, exactly as + * `By_CopyClassConstant` leaves it — this is the value half of that copy */ + stands = By_TwinReplacement(value, constants->twins, constants->types, constants->classes); + if (stands == NULL) stands = value; + return By_NewRef(stands); +} + +/* write the constants into a class namespace, and hand back what was written + * + * the mapping is `{name: value}` for exactly the names the captured body wrote, and it is + * what the class is checked against afterwards */ +static inline PyObject *By_CarryConstants(PyObject *ns, const By_ClassConstants *constants) { + PyObject *carried = PyDict_New(); + Py_ssize_t at; + if (carried == NULL) return NULL; + for (at = 0; constants != NULL && at < constants->count; at++) { + PyObject *value = By_ConstantValue(constants, at); + int failed; + if (value == NULL) continue; + failed = By_SetInNamespace(ns, constants->names[at], value) < 0 + || PyDict_SetItemString(carried, constants->names[at], value) < 0; + Py_DECREF(value); + if (failed) { + Py_DECREF(carried); + return NULL; + } + } + return carried; +} + +/* whether the finished class answers every constant with the object it was handed + * + * writing them into the namespace is what lets the metaclass see them, and it is enough + * for a metaclass that only *reads* one — a `__slots__` an `ABCMeta` passes to + * `type.__new__`, an `_fields` a registry records. it is not enough for one that *makes* + * something of what the body wrote: an `EnumType` handed `STRICT = 'strict'` builds a + * member out of it, and the member is not the value, so every reference the module body + * already took would name the old one. + * + * name for name is what separates the two, and it is asked of the class's own dict — + * which is where a `class` statement's namespace lands, entry for entry, and the only + * place the comparison can be made against the raw object the body wrote. a lookup on the + * class would run the descriptor protocol instead, so a `__class_getitem__ = classmethod(f)` + * would answer a freshly bound method and never be identical to anything. where the check + * fails, the interpreted definition stands — which is the answer such a class had before + * any of this. + * + * that fallback carries the limit every fallback in `By_BuildClass` carries: a twin + * extends the *twin's* base, so a class refused here while a base of this module's is + * emitted would answer `issubclass` False where python answers True. the compile-time + * cascade is what keeps that from arising and the runtime cannot reach as far — the + * choice left here is between the twin and a failed import, and the twin is the better + * of the two. nothing over the stdlib is refused here at all */ +static inline int By_ConstantsHeldUp(PyObject *cls, PyObject *carried) { + PyObject *name, *wanted; + PyObject *own = cls != NULL && PyType_Check(cls) ? ((PyTypeObject *)cls)->tp_dict : NULL; + Py_ssize_t position = 0; + if (carried == NULL) return 1; + while (PyDict_Next(carried, &position, &name, &wanted)) { + int same; + if (own != NULL) { + /* borrowed, and nothing here runs while the walk is open */ + same = PyDict_GetItem(own, name) == wanted; + } else { + /* a metaclass answering with something that is not a class at all */ + PyObject *got = PyObject_GetAttr(cls, name); + if (got == NULL) { + PyErr_Clear(); + return 0; + } + same = got == wanted; + Py_DECREF(got); + } + if (!same) return 0; + } + return 1; +} + /* one entry of a method table, as the descriptor a class namespace holds * * the three cases `type_add_methods` distinguishes: a class method and a static method @@ -1144,8 +1427,8 @@ static inline PyObject *By_MethodDescriptor(PyTypeObject *owner, PyMethodDef *de return PyDescr_NewMethod(owner, def); } -/* the class `meta(name, bases, namespace, **kwds)` builds, with `methods` in that - * namespace +/* the class `meta(name, bases, namespace, **kwds)` builds, with `methods` and + * `constants` in that namespace * * the methods go in *before* the call rather than onto the finished type, and both * halves of that matter. `type.__new__` fills the type slots from the namespace, so a @@ -1153,14 +1436,22 @@ static inline PyObject *By_MethodDescriptor(PyTypeObject *owner, PyMethodDef *de * reads the namespace — an `ABCMeta` deciding which of the base's abstract methods * this class left abstract — sees what the class actually defines. * + * the constants go in for the same reason, and that is the whole of what makes a class + * with one buildable this way: copied onto the *finished* type they would land behind the + * metaclass's back, and a `__slots__` that arrived after `type.__new__` had already given + * the instances a dict is not a `__slots__` at all. what the copy cannot promise, the + * check after the call does — see `By_ConstantsHeldUp` — and where the call does not get + * that far, the raise is the same refusal by another route. + * * the descriptors name `object` as their owner because the type they belong to is what * this call produces. that is also the more faithful answer: the interpreted twin holds * plain functions there, and a plain function checks no receiver either */ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const char *name, PyObject *bases, PyObject *orig_bases, - PyObject *kwds, PyMethodDef *methods) { + PyObject *kwds, PyMethodDef *methods, + const By_ClassConstants *constants) { PyMethodDef *def; - PyObject *module_name, *prepare, *args, *ns, *cls = NULL; + PyObject *module_name, *prepare, *args, *ns, *carried, *cls; PyObject *meta = By_Metaclass(bases, kwds); if (meta == NULL) return NULL; args = Py_BuildValue("(sO)", name, bases); @@ -1211,20 +1502,40 @@ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const cha return NULL; } } + /* after the methods, so that a body writing a name as both leaves the value there — + * which is the answer the check below is made against */ + carried = By_CarryConstants(ns, constants); + if (carried == NULL) { + Py_DECREF(ns); + Py_DECREF(meta); + return NULL; + } args = Py_BuildValue("(sOO)", name, bases, ns); Py_DECREF(ns); - if (args != NULL) { - cls = PyObject_Call(meta, args, kwds); - Py_DECREF(args); + if (args == NULL) { + Py_DECREF(carried); + Py_DECREF(meta); + return NULL; } + cls = PyObject_Call(meta, args, kwds); + Py_DECREF(args); Py_DECREF(meta); + /* the interpreted definition already built this class — the fallback source ran + * before any of this — so a metaclass raising here is the reconstruction being wrong + * rather than the class being unbuildable, and taking the whole import down for it + * would be the worst of the three answers. `ssl`'s `Purpose` is the case: `EnumType` + * is handed a namespace whose members are the twin's finished ones, and building a + * member out of a member raises before the check below could turn it down */ + if (cls == NULL) PyErr_Clear(); /* a `metaclass` that is not a type may hand back anything, and what it hands back is - * what the name means — but it is not a type this module can hang a constant or a - * decorated method on, so the interpreted definition is what stands under it */ - if (cls != NULL && !PyType_Check(cls)) { - Py_DECREF(cls); + * what the name means — but it is not a type this module can hang a decorated method + * on, so the interpreted definition is what stands under it. a class that disagrees + * with what its body wrote is turned down the same way and for the same reason */ + if (cls == NULL || !PyType_Check(cls) || !By_ConstantsHeldUp(cls, carried)) { + Py_XDECREF(cls); cls = By_LookupGlobalString(module_dict, name); } + Py_DECREF(carried); return cls; } @@ -1243,11 +1554,13 @@ static inline PyObject *By_TypeThroughMetaclass(PyObject *module_dict, const cha * layout — so `through_metaclass` is false for a class with fields of its own, and the * interpreted definition the fallback already ran is what answers for it. it is false * again for anything the caller can only put on the *finished* type: a method decorator - * and a class-level constant both land after the metaclass has decided what the class - * defines, and a metaclass that reads its namespace would disagree with them */ + * lands after the metaclass has decided what the class defines, and a metaclass that + * reads its namespace would disagree with it. a class-level constant does not, because + * `constants` carries it into the namespace instead */ static inline PyObject *By_BuildClass(PyObject *module_dict, const char *name, PyObject *bases, PyObject *kwds, PyMethodDef *methods, - PyType_Spec *spec, int through_metaclass) { + PyType_Spec *spec, int through_metaclass, + const By_ClassConstants *constants) { PyObject *cls, *resolved; if (bases == NULL) return NULL; resolved = By_ResolveBases(bases); @@ -1257,13 +1570,14 @@ static inline PyObject *By_BuildClass(PyObject *module_dict, const char *name, } if (spec != NULL && By_SpecTakesBases(resolved)) { cls = PyType_FromSpecWithBases(spec, resolved); - if (cls != NULL && !By_OffsetsHoldUp((PyTypeObject *)cls)) { + if (cls != NULL && !By_OffsetsHoldUp((PyTypeObject *)cls, spec)) { Py_DECREF(cls); cls = By_LookupGlobalString(module_dict, name); } } else if (through_metaclass) { cls = By_TypeThroughMetaclass(module_dict, name, resolved, - resolved == bases ? NULL : bases, kwds, methods); + resolved == bases ? NULL : bases, kwds, methods, + constants); } else { cls = By_LookupGlobalString(module_dict, name); } @@ -1473,28 +1787,6 @@ static inline char By_IsMatchSequence(PyObject *o) { } #endif -/* move a class-level constant from the interpreted definition onto the compiled - * type - * - * a *static* type is immutable to `setattr`, which is what licenses direct - * dispatch — so this writes the type's dict, the way a C extension declares its - * own class attributes - */ -static inline int By_CopyClassConstant(PyObject *module_dict, const char *class_name, - PyTypeObject *type, const char *name) { - PyObject *twin = PyDict_GetItemString(module_dict, class_name); - if (twin == NULL) return 0; - PyObject *value = PyObject_GetAttrString(twin, name); - if (value == NULL) { - PyErr_Clear(); - return 0; - } - int result = PyDict_SetItemString(type->tp_dict, name, value); - Py_DECREF(value); - if (result == 0) PyType_Modified(type); - return result; -} - /* the refusal `__annotations__` gives where a class's own could not be carried across * * `type_get_annotations` hands whatever it finds under that name in `tp_dict` to its own @@ -1873,6 +2165,213 @@ static inline int By_AdoptTwinAttributes(PyObject *const *twins, PyObject *const return 0; } +/* move a class-level constant from the interpreted definition onto the compiled type + * + * a *static* type is immutable to `setattr`, which is what licenses direct dispatch — so + * this writes the type's dict, the way a C extension declares its own class attributes. + * + * the value comes out of the body that definition wrote, so `attr = C` in a class body + * hands over the *interpreted* `C`, and copying that verbatim gives the type an attribute + * naming a class nothing else in the module can reach. a value that *is* a twin is + * therefore replaced by the type standing in for it, exactly as a carried attribute is. + * + * a value that merely *reaches* one is left as the interpreted definition had it, and that + * is the one place this differs from `By_AdoptTwinAttributes`. dropping it instead was + * built and backed out on the measurement: it loses 65 attributes over the corpus — + * `ipaddress` its network constants among them. absence would be the better failure if the + * reach were new, but it is a defect this copy has always had, and a question of its own + * rather than one to settle as a side effect of the identity + */ +static inline int By_CopyClassConstant(PyObject *body, PyTypeObject *type, const char *name, + PyObject *const *twins, PyObject *const *types, + Py_ssize_t count) { + const char *const names[] = {name}; + By_ClassConstants constants = {body, names, 1, twins, types, count}; + /* the same value a class built through its metaclass is handed before the call, so the + * two constructions cannot drift apart about what a constant is */ + PyObject *stands = By_ConstantValue(&constants, 0); + int result; + if (stands == NULL) return 0; + result = PyDict_SetItemString(type->tp_dict, name, stands); + Py_DECREF(stands); + if (result == 0) PyType_Modified(type); + return result; +} + +/* the module-level names still bound to an interpreted twin, moved onto what replaced it + * + * the whole module body runs against the interpreted definitions, so every name it binds + * to a class holds the twin — `Kind = C`, a re-export under another spelling, a name a + * conditional picked — while the compiled type only ever replaces the one name the + * `class` statement wrote. what that leaves is two classes of the same name in the same + * module: `Kind()` builds an object `isinstance(obj, C)` denies, and a compiled method + * handed one refuses it outright with `doesn't apply to a 'C' object`. + * + * a name that *is* a twin is the one shape that can be moved soundly, and it is the same + * substitution `By_TwinReplacement` makes for a carried attribute. it is made against + * whatever now stands under the class's own name rather than against the type directly, + * so a decorated class hands its aliases the decorator's answer — which is what the body + * bound them to — instead of the type the decorator was given. + * + * a value that merely *reaches* a twin is not moved and cannot be: an instance the body + * built has the twin for its type, and a list holding one is the same object the body + * kept. those stay as the body left them */ +static inline int By_RemapTwinAliases(PyObject *module_dict, PyObject *const *twins, + const char *const *names, Py_ssize_t count) { + /* the keys first: the dict is written while this walks, and only for keys it + * already holds, but nothing may run against it mid-walk either way */ + PyObject *keys = PyDict_Keys(module_dict); + Py_ssize_t at; + if (keys == NULL) return -1; + for (at = 0; at < PyList_GET_SIZE(keys); at++) { + PyObject *key = PyList_GET_ITEM(keys, at); + PyObject *value = PyDict_GetItem(module_dict, key); + Py_ssize_t index; + if (value == NULL) continue; + for (index = 0; index < count; index++) { + PyObject *stands; + if (twins[index] == NULL || value != twins[index]) continue; + stands = PyDict_GetItemString(module_dict, names[index]); + /* the class's own name already holds it, and one whose type was never + * installed still holds the twin — neither is a move */ + if (stands == NULL || stands == value) break; + if (PyDict_SetItem(module_dict, key, stands) < 0) { + Py_DECREF(keys); + return -1; + } + break; + } + } + Py_DECREF(keys); + return 0; +} + +/* `__build_class__`, recording what each module-level `class` statement wrote + * + * `state` is `(the real __build_class__, the mapping to record into)`. the class is built + * first and read afterwards, because the namespace itself is never handed back: python + * gives it to the metaclass and to nobody else. what `type.__new__` made of it is the + * closer thing anyway — it is exactly what the interpreted class holds at the moment + * before the first of its decorators is handed it */ +static PyObject *By_CaptureClassBody(PyObject *state, PyObject *args, PyObject *kwds) { + PyObject *cls = PyObject_Call(PyTuple_GET_ITEM(state, 0), args, kwds); + PyObject *name, *qualified, *body; + int outermost; + if (cls == NULL || PyTuple_GET_SIZE(args) < 2 || !PyType_Check(cls)) return cls; + if (((PyTypeObject *)cls)->tp_dict == NULL) return cls; + name = PyTuple_GET_ITEM(args, 1); + /* a class written inside a function can be named the same as one at module level and + * is not the same class. `f..C` against `C` is what tells them apart, and the + * body function python passes here is what carries that qualified name */ + qualified = PyObject_GetAttrString(PyTuple_GET_ITEM(args, 0), "__qualname__"); + if (qualified == NULL) { + PyErr_Clear(); + return cls; + } + outermost = PyObject_RichCompareBool(qualified, name, Py_EQ); + Py_DECREF(qualified); + if (outermost != 1) { + if (outermost < 0) PyErr_Clear(); + return cls; + } + body = PyDict_Copy(((PyTypeObject *)cls)->tp_dict); + /* a body that cannot be recorded is raised out of the `class` statement rather than + * passed over: what would follow is a type carrying no constants at all, and for a + * decorated class that is the defect this capture exists to remove */ + if (body == NULL || PyDict_SetItem(PyTuple_GET_ITEM(state, 1), name, body) < 0) { + Py_XDECREF(body); + Py_DECREF(cls); + return NULL; + } + Py_DECREF(body); + return cls; +} + +/* run a module's fallback source, capturing each class body before its decorators run + * + * the source is the whole interpreted module and it runs to completion before any emitted + * type is built, so by the time a class-level constant is copied onto one, the class it + * would be copied off has already been through its own decorators. a decorator that only + * *reads* the class leaves the value where the body put it, but one that makes something + * of it does not: `@dataclass` deletes the `field(init=False)` a body wrote, and leaves a + * bare `2` where `field(default=2, repr=False)` stood. + * + * so the body is taken while it still is the body. python routes every `class` statement + * through `__build_class__`, and the one a statement reaches is the `__build_class__` of + * *its own frame's* builtins — so a copy of the builtins mapping, put in this module's + * dict, reaches this module's body and nothing else in the process. swapping the entry in + * the real builtins instead would be seen by every thread importing at the same time, + * which on a free-threaded interpreter is a live hazard rather than a theoretical one. + * + * the copy outlives the exec whatever is done with it: python gives a function the + * builtins its defining frame had, so every function this body defines holds this dict for + * as long as it lives. that is why the real entry is put back afterwards rather than the + * dict simply dropped — otherwise a class one of those functions made, at any later point + * in the process, would still be recorded here. + * + * hands back `{name: body}` for the classes the body wrote at module level, as a new + * reference, or NULL with an exception set where the body raised */ +static inline PyObject *By_RunModuleBody(const char *source, PyObject *dict) { + static PyMethodDef capture = {"__build_class__", + (PyCFunction)(void (*)(void))By_CaptureClassBody, + METH_VARARGS | METH_KEYWORDS, NULL}; + PyObject *bodies, *stood, *mapping, *builtins, *real, *state, *wrapper, *result; + int failed; + bodies = PyDict_New(); + if (bodies == NULL) return NULL; + /* an emitted module's dict has no `__builtins__` of its own, and python would then + * give the body's frame the running interpreter's */ + stood = PyDict_GetItemString(dict, "__builtins__"); + if (stood == NULL) stood = PyEval_GetBuiltins(); + Py_XINCREF(stood); + mapping = stood != NULL && PyModule_Check(stood) ? PyModule_GetDict(stood) : stood; + builtins = mapping != NULL && PyDict_Check(mapping) ? PyDict_Copy(mapping) : NULL; + real = builtins == NULL ? NULL : PyDict_GetItemString(builtins, "__build_class__"); + if (real == NULL) { + Py_XDECREF(builtins); + Py_XDECREF(stood); + Py_DECREF(bodies); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, + "no builtins `__build_class__` to run the module body against"); + } + return NULL; + } + Py_INCREF(real); + state = PyTuple_Pack(2, real, bodies); + wrapper = state == NULL ? NULL : PyCFunction_New(&capture, state); + Py_XDECREF(state); + failed = wrapper == NULL || PyDict_SetItemString(builtins, "__build_class__", wrapper) < 0 + || PyDict_SetItemString(dict, "__builtins__", builtins) < 0; + Py_XDECREF(wrapper); + result = failed ? NULL : PyRun_String(source, Py_file_input, dict, dict); + Py_XDECREF(result); + /* whatever the body did, the capture stops here */ + { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + if (PyDict_SetItemString(builtins, "__build_class__", real) < 0 + || PyDict_SetItemString(dict, "__builtins__", stood) < 0) { + PyErr_Clear(); + } + PyErr_Restore(type, value, traceback); + } + Py_DECREF(real); + Py_DECREF(builtins); + Py_DECREF(stood); + if (failed || result == NULL) { + Py_DECREF(bodies); + return NULL; + } + return bodies; +} + +/* the body captured for one class, as a borrowed reference, or NULL where there is none */ +static inline PyObject *By_ClassBody(PyObject *bodies, const char *name) { + if (bodies == NULL) return NULL; + return PyDict_GetItemString(bodies, name); +} + /* the answer a class pattern gives when the attribute it named is simply absent * * a missing attribute is *no match*, not an error — `case Point(z=1):` against a @@ -2837,10 +3336,10 @@ static inline PyObject *By_Method(PyObject *fn) { /* apply a method's decorators, innermost first, to the finished type — which is the * only place a type spec leaves for them. * - * each decorator is looked up in the module namespace, so `@property` and a - * user-defined one resolve the same way. they are folded in memory and the result - * written once, which is also what a class body does: the namespace never holds a - * half-decorated method. `PyType_Modified` is what makes the change visible — the + * each decorator is resolved out of the module namespace by `By_LookupDotted`, so + * `@property` and `@abc.abstractmethod` come out the same way. they are folded in + * memory and the result written once, which is also what a class body does: the + * namespace never holds a half-decorated method. `PyType_Modified` is what makes the change visible — the * attribute cache would otherwise keep serving the undecorated one */ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, const char *owner, const char *name, @@ -2863,7 +3362,7 @@ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, } for (index = count; index > 0; index--) { PyObject *args[1] = {target}; - PyObject *fn = By_LookupGlobalString(dict, decorators[index - 1]); + PyObject *fn = By_LookupDotted(dict, decorators[index - 1]); PyObject *wrapped; if (fn == NULL) { Py_DECREF(target); @@ -2885,9 +3384,10 @@ static inline int By_ApplyMethodDecorators(PyTypeObject *type, PyObject *dict, return 0; } -/* apply `dict[decorator]` to `dict[name]`, in place. this is what lets a - * decorated function still be compiled: the native one goes into the namespace, - * then the decorator wraps it, exactly as the `def` statement would have */ +/* apply the decorator `decorator` names to `dict[name]`, in place. this is what + * lets a decorated function still be compiled: the native one goes into the + * namespace, then the decorator wraps it, exactly as the `def` statement would + * have. `decorator` is a dotted path — see `By_LookupDotted` */ static inline int By_ApplyDecorator(PyObject *dict, const char *name, const char *decorator) { PyObject *target = PyDict_GetItemString(dict, name); if (target == NULL) { @@ -2895,7 +3395,7 @@ static inline int By_ApplyDecorator(PyObject *dict, const char *name, const char return -1; } Py_INCREF(target); - PyObject *fn = By_LookupGlobalString(dict, decorator); + PyObject *fn = By_LookupDotted(dict, decorator); if (fn == NULL) { Py_DECREF(target); return -1; @@ -3197,13 +3697,6 @@ static inline PyObject *By_PackInitKwargs(PyObject *kwds, const char *const *nam return packed; } -/* report every parameter the caller left out, in cpython's own wording. - * - * matching the message matters more than it looks: the differential harness compares - * exception text, and a difference there is a difference a user would see */ -/* every parameter with no default that nothing filled, named the way python names - * them — and positional and keyword-only are counted separately, because python - * reports them in two different sentences */ /* python began qualifying a method by its class in 3.10, so the name the compiler * wrote is trimmed back to its tail on an interpreter that would not have used it */ static inline const char *By_ErrorName(const char *fname) { @@ -3215,6 +3708,16 @@ static inline const char *By_ErrorName(const char *fname) { #endif } +/* every parameter with no default that nothing filled, named the way python names them + * — positional and keyword-only counted separately, because python reports them in two + * different sentences + * + * this is the wording of *last resort*: [`By_Rephrase`] runs first and lets the + * interpreter word the refusal itself, and only a shape it could not build falls back + * to here. so the list joined below is deliberately left as it always was, one comma + * short of python's — a differential test that sees this text is a test whose rephrasing + * never ran, which is the one thing a comparison of two identical strings could not + * otherwise tell anyone */ static inline int By_CheckRequired(const char *const *names, const unsigned char *required, Py_ssize_t count, Py_ssize_t kwonly, PyObject **out, const char *fname) { @@ -3255,30 +3758,162 @@ static inline int By_CheckRequired(const char *const *names, const unsigned char return 0; } +/* a spelling no parameter already has, for a synthetic one that has to be named + * + * the receiver, the `*args` and the `**kwargs` [`By_Rephrase`] writes are named in + * source nothing reads back, so any free spelling does and underscores are appended + * until one is free. free of the *real* names is not on its own enough, though: + * python offers a near miss to a caller who spelled a keyword wrongly, and it draws + * that suggestion from the parameters between the positional-only run and the end of + * the keyword-only one. a `*args` or `**kwargs` name lies outside that range and a + * positional-only one before it, which is why the receiver is written as one */ +static inline void By_SpareName(char *buffer, size_t size, const char *stem, + const char *const *names, Py_ssize_t count) { + size_t used = strlen(stem); + if (used + 1 > size) used = size - 1; + memcpy(buffer, stem, used); + buffer[used] = '\0'; + while (used + 1 < size) { + Py_ssize_t i = 0; + while (i < count && strcmp(buffer, names[i]) != 0) i++; + if (i == count) return; + buffer[used++] = '_'; + buffer[used] = '\0'; + } +} + +/* the caller's positionals with the receiver python counts put back in front of them */ +static inline PyObject *By_ShapeArgs(PyObject *args, Py_ssize_t receiver) { + Py_ssize_t nargs = args == NULL ? 0 : PyTuple_GET_SIZE(args); + Py_ssize_t extra = receiver ? 1 : 0; + PyObject *made = PyTuple_New(nargs + extra); + if (made == NULL) return NULL; + if (extra) PyTuple_SET_ITEM(made, 0, By_NewRef(Py_None)); + for (Py_ssize_t i = 0; i < nargs; i++) { + PyTuple_SET_ITEM(made, i + extra, By_NewRef(PyTuple_GET_ITEM(args, i))); + } + return made; +} + +/* the refusal the interpreter itself would word for a call to a function of this shape + * + * nothing in the c api formats one. `format_missing`, `too_many_positional` and + * `format_kwargs_error` are all static to `ceval.c`, and their wording is fussier than + * it looks: `and` from two names up, a comma *before* that `and` from three up, a range + * rather than a count once any parameter has a default, and a receiver counted in the + * arity sentence but not in the missing-argument one. writing those rules out is what + * left the comma out of this message for the whole of the project's life, and it was + * right when it was written — so the next rule to change would go the same way + * + * so rather than the rules, the *shape*: a python function with the same parameters, + * handed the same call. its body is `pass`, so the only thing the call can do is raise + * what the interpreter raises for the real one. returns 1 having left that exception + * pending, or 0 having left none — which is the two binders disagreeing, and is why the + * caller's own wording stays behind this + * + * the caller's exception must be off the thread before this is reached: it compiles and + * it calls, and neither is reached with one pending */ +static inline int By_Rephrase(const char *const *names, const unsigned char *required, + Py_ssize_t count, Py_ssize_t posonly, Py_ssize_t kwonly, + int variadic, int extras, const char *fname, + Py_ssize_t receiver, PyObject *args, PyObject *kwds) { + char self_name[32], rest_name[32], keys_name[32]; + By_SpareName(self_name, sizeof(self_name), "_by_self", names, count); + By_SpareName(rest_name, sizeof(rest_name), "_by_rest", names, count); + By_SpareName(keys_name, sizeof(keys_name), "_by_keys", names, count); + /* a keyword spelled like the synthetic receiver would bind to it, and the shape + * would then answer about a parameter the real function does not have */ + if (kwds != NULL && receiver && PyDict_GetItemString(kwds, self_name) != NULL) return 0; + + Py_ssize_t limit = count - kwonly; + PyObject *source = PyUnicode_FromString("def _("); + if (receiver) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("%s, ", self_name)); + /* positional-only, so that no near miss is ever offered against it. where the + * function has a positional-only run of its own the marker comes after that */ + if (posonly == 0) PyUnicode_AppendAndDel(&source, PyUnicode_FromString("/, ")); + } + for (Py_ssize_t i = 0; i < limit; i++) { + PyUnicode_AppendAndDel( + &source, PyUnicode_FromFormat("%s%s, ", names[i], required[i] ? "" : "=None")); + if (i + 1 == posonly) PyUnicode_AppendAndDel(&source, PyUnicode_FromString("/, ")); + } + if (variadic) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("*%s, ", rest_name)); + } else if (kwonly > 0) { + PyUnicode_AppendAndDel(&source, PyUnicode_FromString("*, ")); + } + for (Py_ssize_t i = limit; i < count; i++) { + PyUnicode_AppendAndDel( + &source, PyUnicode_FromFormat("%s%s, ", names[i], required[i] ? "" : "=None")); + } + if (extras) PyUnicode_AppendAndDel(&source, PyUnicode_FromFormat("**%s", keys_name)); + PyUnicode_AppendAndDel(&source, PyUnicode_FromString("): pass\n")); + + /* the module namespace the definition lands in, which is also where it is read back + * from. nothing in the source needs a builtin, and evaluation supplies the + * interpreter's own where a namespace carries none */ + PyObject *shape = NULL, *scope = NULL; + const char *text = source == NULL ? NULL : PyUnicode_AsUTF8(source); + if (text != NULL) scope = PyDict_New(); + if (scope != NULL) { + PyObject *ran = PyRun_String(text, Py_file_input, scope, scope); + Py_XDECREF(ran); + if (ran != NULL) shape = PyDict_GetItemString(scope, "_"); + } + int reworded = 0; + if (shape != NULL) { + /* every one of these messages names the *qualified* name, which for a function + * is the one it carries rather than the one its code object was compiled with */ + PyObject *label = PyUnicode_FromString(fname); + int named = label != NULL && PyObject_SetAttrString(shape, "__qualname__", label) == 0; + Py_XDECREF(label); + PyObject *positional = named ? By_ShapeArgs(args, receiver) : NULL; + if (positional != NULL) { + PyObject *answer = PyObject_Call(shape, positional, kwds); + Py_DECREF(positional); + if (answer != NULL) Py_DECREF(answer); + else if (PyErr_ExceptionMatches(PyExc_TypeError)) reworded = 1; + } + } + Py_XDECREF(source); + Py_XDECREF(scope); + /* a shape that could not be built, or one that refused for a reason of its own, + * leaves the thread as it found it */ + if (!reworded) PyErr_Clear(); + return reworded; +} + /* the constructor's binding: the same rules [`By_BindArgs`] applies, read off a tuple * and a dict rather than a fastcall vector — which is the whole of what differs * * `out[i]` receives a *borrowed* pointer, or NULL where the caller supplied nothing * and the default fills it. python counts `self` in its arity message and not in its * missing-argument one, so this does too */ -static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const *names, - Py_ssize_t count, const unsigned char *required, - Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, - int variadic, int extras, const char *fname, int inherited) { +static inline int By_BindInitPlain(PyObject *args, PyObject *kwds, + const char *const *names, Py_ssize_t count, + const unsigned char *required, Py_ssize_t posonly, + Py_ssize_t kwonly, PyObject **out, int variadic, + int extras, const char *fname, int inherited) { fname = By_ErrorName(fname); for (Py_ssize_t i = 0; i < count; i++) out[i] = NULL; Py_ssize_t nargs = args == NULL ? 0 : PyTuple_GET_SIZE(args); /* a keyword-only parameter is one nothing positional can reach, so the run a * caller may fill positionally ends where they begin */ Py_ssize_t positional_limit = count - kwonly; - if (nargs > positional_limit && !variadic) { - /* a class with no `__init__` at all is rejected by `object.__init__`, which - * names the class and does not count a receiver it never had. a *written* - * `def __init__(self)` takes no arguments either and still reports as a method */ - if (inherited) { + /* a class with no `__init__` at all is rejected by `object.__init__`, which names + * the class, does not count a receiver it never had, and asks only whether it was + * given anything — a keyword is as much an excess argument as a positional, and + * saying which one would be a distinction `object_init` never draws. such a class + * takes no parameters at all, so there is nothing else the call could be about. a + * *written* `def __init__(self)` takes no arguments either and still reports as a + * method, which is why this turns on how the class was written and not on `count` */ + if (inherited) { + if (nargs > 0 || (kwds != NULL && PyDict_Size(kwds) > 0)) { PyErr_Format(PyExc_TypeError, "%s() takes no arguments", fname); return -1; } + } else if (nargs > positional_limit && !variadic) { return By_TooManyPositional(fname, required, positional_limit, nargs, 1); } Py_ssize_t positional = nargs < positional_limit ? nargs : positional_limit; @@ -3306,6 +3941,32 @@ static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const return By_CheckRequired(names, required, count, kwonly, out, fname); } +/* the same binding, with a refusal put back into the interpreter's own words + * + * the plain binding writes nothing but `out`, and rewrites all of it before reading any + * — so where the shape declines to reword, running it a second time is how its own + * message comes back, and nothing has to be carried across the attempt */ +static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const *names, + Py_ssize_t count, const unsigned char *required, + Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, + int variadic, int extras, const char *fname, int inherited) { + if (By_BindInitPlain(args, kwds, names, count, required, posonly, kwonly, out, variadic, + extras, fname, inherited) == 0) { + return 0; + } + /* a class that wrote no `__init__` is refused by `object.__init__`, which is not a + * python function and has no shape to model. and a refusal that is not a `TypeError` + * is not an arity one — it is the binding itself having failed */ + if (inherited || !PyErr_ExceptionMatches(PyExc_TypeError)) return -1; + PyErr_Clear(); + if (By_Rephrase(names, required, count, posonly, kwonly, variadic, extras, + By_ErrorName(fname), 1, args, kwds)) { + return -1; + } + return By_BindInitPlain(args, kwds, names, count, required, posonly, kwonly, out, + variadic, extras, fname, inherited); +} + /* bind fastcall arguments to parameter positions, honouring keywords. * * `receiver` is 1 for a method, whose `self` arrives outside the vector but which @@ -3314,11 +3975,12 @@ static inline int By_BindInit(PyObject *args, PyObject *kwds, const char *const * `out[i]` receives a *borrowed* pointer, or NULL where the caller did not supply * that parameter — the wrapper fills those from the defaults. returns -1 with an * exception set on a duplicate, an unexpected name, or too many positionals */ -static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, - const char *const *names, Py_ssize_t count, - const unsigned char *required, Py_ssize_t posonly, - Py_ssize_t kwonly, PyObject **out, int variadic, int extras, - const char *fname, Py_ssize_t receiver) { +static inline int By_BindArgsPlain(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames, const char *const *names, + Py_ssize_t count, const unsigned char *required, + Py_ssize_t posonly, Py_ssize_t kwonly, PyObject **out, + int variadic, int extras, const char *fname, + Py_ssize_t receiver) { fname = By_ErrorName(fname); /* a keyword-only parameter is one nothing positional can reach, so the run a * caller may fill positionally ends where they begin */ @@ -3351,6 +4013,54 @@ static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject return By_CheckRequired(names, required, count, kwonly, out, fname); } +/* a fastcall vector as the tuple and dict a plain call takes. only the error path needs + * either, and it is cold */ +static inline PyObject *By_VectorTuple(PyObject *const *args, Py_ssize_t nargs) { + PyObject *made = PyTuple_New(nargs); + if (made == NULL) return NULL; + for (Py_ssize_t i = 0; i < nargs; i++) PyTuple_SET_ITEM(made, i, By_NewRef(args[i])); + return made; +} + +static inline PyObject *By_VectorKwds(PyObject *const *args, Py_ssize_t nargs, + PyObject *kwnames) { + PyObject *made = PyDict_New(); + if (made == NULL || kwnames == NULL) return made; + Py_ssize_t keywords = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t k = 0; k < keywords; k++) { + if (PyDict_SetItem(made, PyTuple_GET_ITEM(kwnames, k), args[nargs + k]) < 0) { + Py_DECREF(made); + return NULL; + } + } + return made; +} + +/* the same binding, with a refusal put back into the interpreter's own words */ +static inline int By_BindArgs(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, + const char *const *names, Py_ssize_t count, + const unsigned char *required, Py_ssize_t posonly, + Py_ssize_t kwonly, PyObject **out, int variadic, int extras, + const char *fname, Py_ssize_t receiver) { + if (By_BindArgsPlain(args, nargs, kwnames, names, count, required, posonly, kwonly, out, + variadic, extras, fname, receiver) == 0) { + return 0; + } + if (!PyErr_ExceptionMatches(PyExc_TypeError)) return -1; + PyErr_Clear(); + PyObject *tuple = By_VectorTuple(args, nargs); + PyObject *dict = tuple == NULL ? NULL : By_VectorKwds(args, nargs, kwnames); + int reworded = dict != NULL + && By_Rephrase(names, required, count, posonly, kwonly, variadic, extras, + By_ErrorName(fname), receiver, tuple, dict); + Py_XDECREF(tuple); + Py_XDECREF(dict); + if (reworded) return -1; + PyErr_Clear(); + return By_BindArgsPlain(args, nargs, kwnames, names, count, required, posonly, kwonly, + out, variadic, extras, fname, receiver); +} + /* `with EXPR`: the manager's `__enter__`, looked up on the *type* the way the * interpreter does rather than on the instance */ /* `__aenter__` and `__aexit__`, which hand back *awaitables* rather than answers diff --git a/crates/by_rt/src/lib.rs b/crates/by_rt/src/lib.rs index 4bad569f38..32210b1036 100644 --- a/crates/by_rt/src/lib.rs +++ b/crates/by_rt/src/lib.rs @@ -38,8 +38,39 @@ mod tests { "By_ObjCompare", "By_Truthy", "By_ApplyMethodDecorators", + "By_SpecClass", + "By_SpecSubclass", ] { assert!(BY_H.contains(symbol), "the runtime is missing {symbol}"); } } + + /// both constructions for a class whose fields sit past a base's instance answer + /// with nothing rather than with something else, because the caller's only move is + /// to leave the whole module as its interpreted definition already built it. one + /// that raised instead would make module init fail the import + #[test] + fn a_layout_that_does_not_hold_up_is_refused_rather_than_raised() { + for construction in ["By_SpecClass", "By_SpecSubclass"] { + let body = BY_H + .split_once(&format!("static inline PyObject *{construction}(")) + .expect("the construction is defined") + .1 + .split_once("\n}\n") + .expect("it ends") + .0; + assert!( + body.contains("if (!By_OffsetsHoldUp((PyTypeObject *)cls"), + "{construction} checks the finished type's offsets" + ); + assert!( + !body.contains("PyErr_Set") && !body.contains("PyErr_Format"), + "{construction} raises nothing: {body}" + ); + assert!( + body.contains("return NULL;"), + "{construction} answers with nothing" + ); + } + } } From c0eee2614031c3446aac273d271f23b0dec902b6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:35:44 +1000 Subject: [PATCH 4/7] a subscript is a type application when the thing subscripted is a type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `sys.modules[type(point).__module__]` was emitted as `sys.modules[str]` — the key replaced by the type inferred for it, which raises at runtime. for `base.attr[…]` the walker asked whether the *base* was a module, where the `Name` arm beside it asked the exact question: is this a type? `typing.List` and `sys.modules` are both attributes of a module, so the weak question could not tell them apart, and the slice of an ordinary dict lookup was walked as a type expression. one predicate now asks that of whatever is being subscripted, with a `trailing_name` helper for the nine sites that also need a particular name. the two questions stay separate on purpose: the name alone cannot tell `typing.List` from an unrelated `mine.List`, and the type alone cannot tell `Callable` from any other generic. three further sites carried the same latent bug and are fixed by the same change. --- .../src/reverse_transforms/callable.rs | 19 +----- .../src/reverse_transforms/intersection.rs | 13 +---- .../src/reverse_transforms/literal_types.rs | 14 +---- .../src/reverse_transforms/not_type.rs | 11 +--- .../src/reverse_transforms/subscript.rs | 11 +--- .../src/reverse_transforms/tuple_type.rs | 14 ++--- .../src/reverse_transforms/type_is.rs | 11 +--- .../src/reverse_transforms/unpack.rs | 11 +--- .../src/transforms/literal_types.rs | 20 +------ .../src/transforms/symbolic_type_op.rs | 51 ++++++++++++++++ .../src/transforms/type_expr_walker.rs | 11 +--- .../src/transforms/type_reification.rs | 14 ++--- crates/by_transforms/src/type_info.rs | 58 ++++++++++++------- 13 files changed, 117 insertions(+), 141 deletions(-) diff --git a/crates/by_transforms/src/reverse_transforms/callable.rs b/crates/by_transforms/src/reverse_transforms/callable.rs index cdc98a7268..d5cccd8977 100644 --- a/crates/by_transforms/src/reverse_transforms/callable.rs +++ b/crates/by_transforms/src/reverse_transforms/callable.rs @@ -17,7 +17,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct CallableReverse<'src> { types: &'src dyn TypeInfo, @@ -55,24 +55,11 @@ impl<'src> CallableReverse<'src> { } fn is_callable_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Callable" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Callable" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Callable") && self.types.subscript_is_type_context(expr) } fn is_type_context_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// rewrite the punctuation of `Callable[, ]` into the arrow diff --git a/crates/by_transforms/src/reverse_transforms/intersection.rs b/crates/by_transforms/src/reverse_transforms/intersection.rs index 7cebaba943..5aa516316f 100644 --- a/crates/by_transforms/src/reverse_transforms/intersection.rs +++ b/crates/by_transforms/src/reverse_transforms/intersection.rs @@ -9,7 +9,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct IntersectionReverse<'src> { source: &'src str, @@ -31,16 +31,7 @@ impl<'src> IntersectionReverse<'src> { } fn is_intersection_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => { - n.id.as_str() == "Intersection" && self.types.subscript_is_type_context(n) - } - Expr::Attribute(a) => { - a.attr.id.as_str() == "Intersection" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Intersection") && self.types.subscript_is_type_context(expr) } fn rewrite(&mut self, expr: &Expr) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/literal_types.rs b/crates/by_transforms/src/reverse_transforms/literal_types.rs index c67bbe8399..2465ca1ccf 100644 --- a/crates/by_transforms/src/reverse_transforms/literal_types.rs +++ b/crates/by_transforms/src/reverse_transforms/literal_types.rs @@ -14,7 +14,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, ExprSubscript, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralReverse<'src> { source: &'src str, @@ -38,17 +38,7 @@ impl<'src> LiteralReverse<'src> { /// `Literal` or `typing.Literal` / `typing_extensions.Literal`, where the /// bare name resolves to an import or is unresolved fn is_literal_name(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => n.id.as_str() == "Literal" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Literal" - && match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - } - } - _ => false, - } + trailing_name(value) == Some("Literal") && self.types.subscript_is_type_context(value) } fn rewrite_literal_subscript(&self, s: &ExprSubscript) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/not_type.rs b/crates/by_transforms/src/reverse_transforms/not_type.rs index f451c34141..233515a989 100644 --- a/crates/by_transforms/src/reverse_transforms/not_type.rs +++ b/crates/by_transforms/src/reverse_transforms/not_type.rs @@ -8,7 +8,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Operator, Stmt}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct NotTypeReverse<'src> { source: &'src str, @@ -30,14 +30,7 @@ impl<'src> NotTypeReverse<'src> { } fn is_not_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Not" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Not" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Not") && self.types.subscript_is_type_context(expr) } fn rewrite(&mut self, expr: &Expr) -> Option { diff --git a/crates/by_transforms/src/reverse_transforms/subscript.rs b/crates/by_transforms/src/reverse_transforms/subscript.rs index 9109a0642a..63246e1ff7 100644 --- a/crates/by_transforms/src/reverse_transforms/subscript.rs +++ b/crates/by_transforms/src/reverse_transforms/subscript.rs @@ -15,7 +15,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct SubscriptReverse<'src> { source: &'src str, @@ -33,14 +33,7 @@ impl<'src> SubscriptReverse<'src> { } fn is_type_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } } diff --git a/crates/by_transforms/src/reverse_transforms/tuple_type.rs b/crates/by_transforms/src/reverse_transforms/tuple_type.rs index c10978134b..b86b4f3046 100644 --- a/crates/by_transforms/src/reverse_transforms/tuple_type.rs +++ b/crates/by_transforms/src/reverse_transforms/tuple_type.rs @@ -16,7 +16,7 @@ use ruff_python_ast::visitor::Visitor; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct TupleTypeReverse<'src> { source: &'src str, @@ -58,19 +58,15 @@ impl<'src> TupleTypeReverse<'src> { fn is_tuple_name(&self, expr: &Expr) -> bool { match expr { - Expr::Name(n) => n.id.as_str() == "tuple" && self.types.subscript_is_type_context(n), + Expr::Name(_) => { + trailing_name(expr) == Some("tuple") && self.types.subscript_is_type_context(expr) + } _ => false, } } fn is_type_context_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// `*tuple[T, ...]` as a tuple element round-trips to the basedpython diff --git a/crates/by_transforms/src/reverse_transforms/type_is.rs b/crates/by_transforms/src/reverse_transforms/type_is.rs index 58bf1de721..6eccbc1829 100644 --- a/crates/by_transforms/src/reverse_transforms/type_is.rs +++ b/crates/by_transforms/src/reverse_transforms/type_is.rs @@ -10,7 +10,7 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct TypeIsReverse<'src> { source: &'src str, @@ -32,14 +32,7 @@ impl<'src> TypeIsReverse<'src> { } fn is_type_is(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "TypeIs" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "TypeIs" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("TypeIs") && self.types.subscript_is_type_context(expr) } } diff --git a/crates/by_transforms/src/reverse_transforms/unpack.rs b/crates/by_transforms/src/reverse_transforms/unpack.rs index 0bc0319305..a45b79247d 100644 --- a/crates/by_transforms/src/reverse_transforms/unpack.rs +++ b/crates/by_transforms/src/reverse_transforms/unpack.rs @@ -11,7 +11,7 @@ use ruff_python_ast::visitor::{Visitor, walk_stmt}; use ruff_python_ast::{Expr, Stmt}; use ruff_text_size::{Ranged, TextRange, TextSize}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct UnpackReverse<'src> { source: &'src str, @@ -29,14 +29,7 @@ impl<'src> UnpackReverse<'src> { } fn is_unpack_name(&self, expr: &Expr) -> bool { - match expr { - Expr::Name(n) => n.id.as_str() == "Unpack" && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == "Unpack" - && matches!(a.value.as_ref(), Expr::Name(n) if self.types.attr_base_is_type_context(n)) - } - _ => false, - } + trailing_name(expr) == Some("Unpack") && self.types.subscript_is_type_context(expr) } fn process_vararg_annotation(&mut self, ann: &Expr) { diff --git a/crates/by_transforms/src/transforms/literal_types.rs b/crates/by_transforms/src/transforms/literal_types.rs index a7d0f965d6..160bbe4074 100644 --- a/crates/by_transforms/src/transforms/literal_types.rs +++ b/crates/by_transforms/src/transforms/literal_types.rs @@ -17,7 +17,7 @@ use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::type_expr_walker::{ Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping, }; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; pub(crate) struct LiteralType<'src> { source: &'src str, @@ -43,26 +43,12 @@ impl<'src> LiteralType<'src> { /// Whether a `Subscript.value` resolves to something whose subscript slice /// is a type-argument position. fn is_type_subscript(&self, value: &Expr) -> bool { - match value { - Expr::Name(n) => self.types.subscript_is_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && self.types.subscript_is_type_context(value) } /// Is `value` a reference to the named typing special form? fn is_typing_name(&self, value: &Expr, name: &str) -> bool { - match value { - Expr::Name(n) => n.id.as_str() == name && self.types.subscript_is_type_context(n), - Expr::Attribute(a) => { - a.attr.id.as_str() == name - && matches!(a.value.as_ref(), Expr::Name(base) if self.types.attr_base_is_type_context(base)) - } - _ => false, - } + trailing_name(value) == Some(name) && self.types.subscript_is_type_context(value) } fn is_annotated_name(&self, value: &Expr) -> bool { diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index d03022ea35..4d35e7ad13 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -566,6 +566,57 @@ mod tests { check("x = 1 + 1\n", "x = 1 + 1\n"); } + #[test] + fn a_module_attribute_that_is_not_a_type_keeps_its_subscript_key() { + // `sys.modules` is a dict, so `sys.modules[k]` is a runtime lookup and + // `k` is a value. asking only whether the *base* (`sys`) is a module + // said yes here as readily as it does for `typing.List`, and the key + // was then folded to the type inferred for it — `sys.modules[str]`, + // which raises at runtime + check( + indoc! {" + import sys + + + class Point: + x: int + + + def main(): + point = Point() + print(sys.modules[type(point).__module__].__name__) + "}, + indoc! {" + import sys + + + class Point: + x: int + + + def main(): + point = Point() + print(sys.modules[type(point).__module__].__name__) + if __name__ == \"__main__\": + main() + "}, + ); + } + + #[test] + fn a_module_attribute_that_is_a_type_still_takes_type_arguments() { + // the other side of the same question: `typing.List` *is* a type, so its + // slice stays a type position and the fold still happens there + check( + "import typing\nc: typing.List[1 + 1]\n", + indoc! {" + from typing import Literal + import typing + c: typing.List[Literal[2]] + "}, + ); + } + #[test] fn attribute_type_folds_to_the_bound_member() { check_py312( diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 5b3defefd8..101903c6a0 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -38,7 +38,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Operator, Parameters, Stmt, TypeParam, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; /// the kind of type position currently being visited. lets visitors /// distinguish (e.g.) a syntactic annotation from an interior subtree @@ -283,14 +283,7 @@ impl TypePosWalker<'_> { let Some(types) = self.types else { return false; }; - match value { - Expr::Name(n) => types.subscript_is_known_type_context(n), - Expr::Attribute(a) => match a.value.as_ref() { - Expr::Name(base) => types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - } + trailing_name(value).is_some() && types.subscript_is_known_type_context(value) } } diff --git a/crates/by_transforms/src/transforms/type_reification.rs b/crates/by_transforms/src/transforms/type_reification.rs index f51d10325e..3e50a6ab16 100644 --- a/crates/by_transforms/src/transforms/type_reification.rs +++ b/crates/by_transforms/src/transforms/type_reification.rs @@ -41,7 +41,7 @@ use ruff_python_ast::{self as ast, Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; -use crate::type_info::TypeInfo; +use crate::type_info::{TypeInfo, trailing_name}; /// dunder assignments whose values static readers (linters, type checkers, /// dataclass machinery) require to stay literal displays @@ -121,14 +121,10 @@ impl<'ast> Visitor<'ast> for Reifier<'_> { // in a type-context subscript (`dict[str, int]`, legacy // `Callable[[int], str]`) the slice is a type expression, and // a display there is type syntax, not a value - let type_context = match subscript.value.as_ref() { - Expr::Name(name) => self.types.subscript_is_type_context(name), - Expr::Attribute(attribute) => match attribute.value.as_ref() { - Expr::Name(base) => self.types.attr_base_is_type_context(base), - _ => false, - }, - _ => false, - }; + let type_context = trailing_name(subscript.value.as_ref()).is_some() + && self + .types + .subscript_is_type_context(subscript.value.as_ref()); if type_context { self.visit_expr(&subscript.value); } else { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 8977cf6028..97ecbf5e16 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -71,19 +71,23 @@ pub(crate) enum CaptureKind { } pub(crate) trait TypeInfo { - /// whether `X[…]` where `X` is `name` treats the slice as type arguments. - /// returns `true` for unresolved / unknown names (covers builtins like - /// `list`, unimported sugar like `Union`) - fn subscript_is_type_context(&self, name: &ExprName) -> bool; - - /// stricter variant: only `true` when ty *resolved* `name` to a class / - /// generic / special form. unresolved names return `false`. used by + /// whether `X[…]` treats the slice as type arguments, where `X` is whatever + /// is being subscripted — a bare name (`list[int]`) or a dotted one + /// (`typing.List[int]`). returns `true` for unresolved / unknown values + /// (covers builtins like `list`, unimported sugar like `Union`) + /// + /// the question is asked of `X` itself rather than of its base. asking the + /// base "is this a module?" answers a different and much weaker question: + /// `typing.List` and `sys.modules` are both attributes of a module, but only + /// the first is a type — and reading `sys.modules[key]` as a type + /// application rewrote the key to the type it inferred for it + fn subscript_is_type_context(&self, value: &Expr) -> bool; + + /// stricter variant: only `true` when ty *resolved* the value to a class / + /// generic / special form. unresolved values return `false`. used by /// transforms that may fire on value-position subscripts (where an /// unresolved name should be treated as a runtime subscript, not a type) - fn subscript_is_known_type_context(&self, name: &ExprName) -> bool; - - /// whether `base.attr[…]` (base = a module or class) treats slice as type args - fn attr_base_is_type_context(&self, base: &ExprName) -> bool; + fn subscript_is_known_type_context(&self, value: &Expr) -> bool; fn is_function(&self, name: &ExprName) -> bool; @@ -543,8 +547,8 @@ impl TypeInfo for SemanticModel<'_> { .is_some_and(|ty| ty.is_attribute_type(self.db())) } - fn subscript_is_type_context(&self, name: &ExprName) -> bool { - match name.inferred_type(self) { + fn subscript_is_type_context(&self, value: &Expr) -> bool { + match value.inferred_type(self) { Some(ty) => ty.is_subscript_type_context(), // unresolved → assume type context (covers builtins like `list`, // unknown imports, basedpython sugar contexts) @@ -552,20 +556,13 @@ impl TypeInfo for SemanticModel<'_> { } } - fn subscript_is_known_type_context(&self, name: &ExprName) -> bool { - match name.inferred_type(self) { + fn subscript_is_known_type_context(&self, value: &Expr) -> bool { + match value.inferred_type(self) { Some(ty) => ty.is_subscript_type_context() && !ty.is_dynamic(), None => false, } } - fn attr_base_is_type_context(&self, base: &ExprName) -> bool { - match base.inferred_type(self) { - Some(ty) => ty.is_module_or_type(), - None => true, - } - } - fn is_function(&self, name: &ExprName) -> bool { name.inferred_type(self) .is_some_and(|ty| ty.as_function_literal().is_some()) @@ -1178,6 +1175,23 @@ impl TypeInfo for SemanticModel<'_> { /// diagnostic, where a symbolic arithmetic operation is shown as the expression it stands /// for (`I + 1`); emitting that would evaluate `_I + 1` on a `TypeVar` object at import, so /// the transpiler asks for the type it reduces to instead +/// the last component of a name or a dotted name — `List` for both `List` and +/// `typing.List` +/// +/// every spelling that takes type arguments is written one of those two ways, so +/// a transform looking for one asks for the trailing name and then asks +/// [`TypeInfo::subscript_is_type_context`] whether the whole thing is a type. the +/// two questions are separate on purpose: the name alone cannot tell +/// `typing.List` from an unrelated `mine.List`, and the type alone cannot tell +/// `Callable` from any other generic +pub(crate) fn trailing_name(expr: &Expr) -> Option<&str> { + match expr { + Expr::Name(name) => Some(name.id.as_str()), + Expr::Attribute(attribute) => Some(attribute.attr.id.as_str()), + _ => None, + } +} + fn display_for_python<'db>( db: &'db dyn Db, env: &ProgramEnvironment<'db>, From 3d6e8138288d24467c01bd3277fb3719545e8081 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:35:44 +1000 Subject: [PATCH 5/7] the sweeps exercise what they walk, and say what they measured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a module was staged alone, so `from . import x` had nothing to resolve against and about a hundred of the 550 reported `import-failed` on both legs — walked but never exercised. a package member is now staged inside a copy of its package, under one outer package nobody else names: a copy under the real name would stand in for the interpreter's own, and `encodings` is already in `sys.modules` before a probe starts. that recovered 89 modules, and a per-stage exclude negation recovered `venv`, the last one ty's own defaults hid. `isoimport` scored a killed leg as agreement, because a leg killed by the import alarm prints exactly what a clean import prints — nothing. the exit status decides now, and `timed-out` and `died` are their own categories rather than silence. a sweep also pays macos's dylib check through `ctypes.CDLL` before it starts timing: that cost is 0.42s idle and 17.8s under contention, it landed inside the import bound, and it came back as `timed-out` — 26 of them in one run, all of which passed when re-run. `CDLL` does not call `PyInit_`, so nothing runs twice. two tools rather than two rules. `scripts/bg.sh` runs a long job and polls it, telling finished-with-status from still-running from *killed*, by recorded pid rather than by pattern — hand-rolled waiting was measured at 58% of all wall time across five agent runs, a third of it in `pgrep -f` loops matching their own command line, which never return. and `scripts/shrink.py` minimises a reproducer without fooling itself: minimising is what cracked every cycle panic here, and three separate traps each produced a confident wrong answer before being guarded — the worst turning 2714 lines into 4 that reproduce nothing. --- scripts/bg.sh | 112 ++ scripts/native-bench/README.md | 19 + scripts/native-bench/bench.py | 1074 +++++++++++++++++ scripts/native-bench/programs.toml | 150 +++ scripts/native-bench/programs/alloc.py | 27 + scripts/native-bench/programs/bigint.py | 33 + scripts/native-bench/programs/calls.py | 24 + scripts/native-bench/programs/chars.py | 37 + scripts/native-bench/programs/coro.py | 40 + scripts/native-bench/programs/dictget.py | 33 + scripts/native-bench/programs/dicthist.py | 34 + scripts/native-bench/programs/dot.py | 31 + scripts/native-bench/programs/excs.py | 55 + scripts/native-bench/programs/fields.py | 27 + scripts/native-bench/programs/gen.py | 30 + scripts/native-bench/programs/generic.py | 32 + scripts/native-bench/programs/generic_mono.py | 31 + scripts/native-bench/programs/inherit.py | 44 + scripts/native-bench/programs/keybuild.py | 22 + scripts/native-bench/programs/loops.py | 30 + scripts/native-bench/programs/mandel.py | 38 + .../native-bench/programs/mandel_inline.py | 35 + scripts/native-bench/programs/methods.py | 28 + scripts/native-bench/programs/objects.py | 30 + scripts/native-bench/programs/prefix.py | 29 + scripts/native-bench/programs/recurse.py | 16 + scripts/native-bench/programs/sets.py | 33 + scripts/native-bench/programs/sieve.py | 30 + scripts/native-bench/programs/strops.py | 29 + scripts/native-bench/programs/tuples.py | 25 + scripts/native-bench/programs/words.py | 19 + scripts/native-bench/timer.py | 165 +++ scripts/native-sweeps/buildsweep.sh | 4 +- scripts/native-sweeps/census.sh | 25 + scripts/native-sweeps/instancecensus.sh | 32 +- scripts/native-sweeps/isoconstruct.sh | 112 +- scripts/native-sweeps/isoimport.sh | 54 +- scripts/native-sweeps/isosubclass.sh | 112 +- scripts/native-sweeps/isosurface.sh | 64 +- scripts/native-sweeps/sweeplib.sh | 237 +++- scripts/shrink.py | 184 +++ 41 files changed, 3046 insertions(+), 140 deletions(-) create mode 100755 scripts/bg.sh create mode 100644 scripts/native-bench/README.md create mode 100644 scripts/native-bench/bench.py create mode 100644 scripts/native-bench/programs.toml create mode 100644 scripts/native-bench/programs/alloc.py create mode 100644 scripts/native-bench/programs/bigint.py create mode 100644 scripts/native-bench/programs/calls.py create mode 100644 scripts/native-bench/programs/chars.py create mode 100644 scripts/native-bench/programs/coro.py create mode 100644 scripts/native-bench/programs/dictget.py create mode 100644 scripts/native-bench/programs/dicthist.py create mode 100644 scripts/native-bench/programs/dot.py create mode 100644 scripts/native-bench/programs/excs.py create mode 100644 scripts/native-bench/programs/fields.py create mode 100644 scripts/native-bench/programs/gen.py create mode 100644 scripts/native-bench/programs/generic.py create mode 100644 scripts/native-bench/programs/generic_mono.py create mode 100644 scripts/native-bench/programs/inherit.py create mode 100644 scripts/native-bench/programs/keybuild.py create mode 100644 scripts/native-bench/programs/loops.py create mode 100644 scripts/native-bench/programs/mandel.py create mode 100644 scripts/native-bench/programs/mandel_inline.py create mode 100644 scripts/native-bench/programs/methods.py create mode 100644 scripts/native-bench/programs/objects.py create mode 100644 scripts/native-bench/programs/prefix.py create mode 100644 scripts/native-bench/programs/recurse.py create mode 100644 scripts/native-bench/programs/sets.py create mode 100644 scripts/native-bench/programs/sieve.py create mode 100644 scripts/native-bench/programs/strops.py create mode 100644 scripts/native-bench/programs/tuples.py create mode 100644 scripts/native-bench/programs/words.py create mode 100644 scripts/native-bench/timer.py create mode 100755 scripts/native-sweeps/census.sh create mode 100755 scripts/shrink.py diff --git a/scripts/bg.sh b/scripts/bg.sh new file mode 100755 index 0000000000..eb58781312 --- /dev/null +++ b/scripts/bg.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# start a long job and wait for it across several turns, without burning the +# 600-second command bound on a wait that answers nothing +# +# a build, a test run or a sweep outlives the bound the tool harness puts on one +# command, so it has to run in the background and be polled. hand-rolled polling +# has been measured as **the single largest cost in this work**: across five +# agents and 12h27m of wall time, 7h13m went into `sleep`/`until` loops against +# 43m of `cargo build`. 3h of that was loops that ran to the bound and returned +# nothing, and 2h was loops that could never have returned at all: +# +# until ! pgrep -f "native-sweeps/isoimport.sh"; do sleep 30; done +# +# `pgrep -f` matches against whole command lines, and that loop's own command +# line contains the pattern — so it matches itself, and waits for the bound no +# matter what the sweep is doing. this file exists so nobody writes that again +# +# usage: +# scripts/bg.sh start launch, return immediately +# scripts/bg.sh wait [seconds] wait up to N (default 540), then +# report `running` and return 0 so +# the caller can simply call again +# scripts/bg.sh status done/running/missing, no waiting +# scripts/bg.sh log [lines] tail the output +# +# `wait` distinguishes the two outcomes a bare sleep loop cannot: `done rc=N` +# means the job really finished and N is its exit status; `running` means it is +# still going and nothing is wrong. a job that dies still writes its marker, so +# a crash is never mistaken for slowness + +set -u + +dir() { + # the scratchpad if the harness gave us one, else a temp dir. keeping the + # markers out of the project matters: `by compile` compiles every source it + # finds beside the file it was given + printf '%s\n' "${BY_BG_DIR:-${TMPDIR:-/tmp}/by-bg}" +} + +start() { + local name="$1"; shift + local d; d=$(dir); mkdir -p "$d" + rm -f "$d/$name.done" "$d/$name.log" "$d/$name.pid" + # the marker is written by the same shell that runs the command, after it + # exits, so it cannot be missed and it carries the real status + # + # the single quotes are the point: `$@`, `$?` and `$0` are for the *inner* shell to + # expand once it has run the command, not for this one to expand now + # shellcheck disable=SC2016 + nohup bash -c '"$@" ; printf "%s\n" "$?" > "$0"' \ + "$d/$name.done" "$@" > "$d/$name.log" 2>&1 & + printf '%s\n' "$!" > "$d/$name.pid" + printf 'started %s (log %s)\n' "$name" "$d/$name.log" +} + +# a job killed outright — OOM, SIGKILL, the machine giving up — never reaches the +# line that writes its marker, and without this it would report `running` until +# the caller gave up. so liveness is checked too, and by *pid*: `kill -0` takes a +# number, which cannot match the command line of the shell asking the question. +# that is the whole difference from `pgrep -f`, which can and does +status() { + local name="$1" d; d=$(dir) + if [ -f "$d/$name.done" ]; then + printf 'done rc=%s\n' "$(cat "$d/$name.done")" + return 0 + fi + if [ ! -f "$d/$name.log" ]; then + printf 'missing\n' + return 0 + fi + local pid="" + [ -f "$d/$name.pid" ] && pid=$(cat "$d/$name.pid") + if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then + # the marker is written *before* the shell exits, so seeing the exit first + # only means the write has not landed yet. re-read once before calling it a + # death, or a job that merely finished quickly is reported as killed + sleep 1 + if [ -f "$d/$name.done" ]; then + printf 'done rc=%s\n' "$(cat "$d/$name.done")" + return 0 + fi + printf 'died (no exit status — killed, not finished)\n' + return 0 + fi + printf 'running\n' +} + +wait_for() { + local name="$1" bound="${2:-540}" d; d=$(dir) + local waited=0 state + while [ "$waited" -lt "$bound" ]; do + state=$(status "$name") + case "$state" in + running) ;; + *) printf '%s\n' "$state"; return 0 ;; + esac + sleep 5 + waited=$((waited + 5)) + done + # not a failure — the caller polls again. returning 0 keeps a bounded wait + # from reading as a broken command + printf 'running (%ss elapsed, call again)\n' "$waited" + return 0 +} + +case "${1:-}" in + start) shift; start "$@" ;; + wait) shift; wait_for "$@" ;; + status) shift; status "$@" ;; + log) shift; tail -n "${2:-40}" "$(dir)/$1.log" ;; + *) printf 'usage: %s {start|wait|status|log} [...]\n' "$0" >&2; exit 2 ;; +esac diff --git a/scripts/native-bench/README.md b/scripts/native-bench/README.md new file mode 100644 index 0000000000..5b6c50e597 --- /dev/null +++ b/scripts/native-bench/README.md @@ -0,0 +1,19 @@ +# native-bench + +```sh +cargo build --release --bin by +uv run --no-project --python 3.13 python scripts/native-bench/bench.py +uv run --no-project --python 3.13 python scripts/native-bench/bench.py --self-check +``` + +times each benchmark four ways — interpreted, `by compile`, `by compile` again, +and mypyc — and reports paired ratios with the noise floor those two identical +builds measured for themselves + +the method, the guardrails and the reason for every benchmark in the set are in +[the docs](../../docs/basedpython/development/compilation/benchmarks.md). read +that before changing anything here, and before quoting a number out of it + +`--python` takes anything `uv python find` accepts, including a free-threaded +build (`3.13t`, `3.14t`) — the version is recorded in the run's metadata and a +baseline from a different one warns before it compares diff --git a/scripts/native-bench/bench.py b/scripts/native-bench/bench.py new file mode 100644 index 0000000000..884b2e4dcb --- /dev/null +++ b/scripts/native-bench/bench.py @@ -0,0 +1,1074 @@ +"""the native benchmark suite: stage, build, prove, time, compare + + scripts/native-bench/bench.py # everything + scripts/native-bench/bench.py mandel dot # only these + scripts/native-bench/bench.py --json today.json + scripts/native-bench/bench.py --baseline last-week.json + +the method it enforces is written up in +`docs/basedpython/development/compilation/benchmarks.md`. the short version: + +- **the ratio is the measurement.** absolute times on this suite move by a + factor of four with machine load, so nothing is reported from one build's + clock alone. every ratio is *paired* — the four builds of a benchmark are + timed in one process, round by round, so a spike lands on all four and mostly + cancels in the quotient +- **the table carries its own error bar.** every benchmark is built twice by the + same compiler, and the second build is timed alongside the first. two builds + of the same source are the same program, so the ratio between them is the + suite's noise floor for that benchmark, measured rather than assumed. a + difference smaller than the floor is not a difference +- **nothing is timed until it has proved what it is.** a compiled build has to + import as a real extension module, from a file inside this run's own root, + newer than this run's build, and it has to return the same answer as the + interpreted one. every one of those is a refusal rather than a warning +- **a decline is a failure, not a footnote.** `programs.toml` records how many + functions each benchmark is expected to leave interpreted, and a run where the + count moved either way fails. otherwise a benchmark can quietly stop measuring + compiled code and go on posting numbers +- **the harness cannot match nothing.** an unknown benchmark name, a program + with no manifest entry, a manifest entry with no program, an empty selection: + all of them exit non-zero. a measurement harness that cannot fail loudly is + worse than none +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import platform +import shutil +import statistics +import subprocess +import sys +import tempfile +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +PROGRAMS = HERE / "programs" +MANIFEST = HERE / "programs.toml" + +# one sample is as many calls as it takes to land between these, measured +# against the fastest and the slowest build respectively. every build of a +# benchmark runs the same number, so the pairing stays exact +MIN_SAMPLE = 0.010 +MAX_SAMPLE = 0.150 +PROBE_SAMPLE = 0.010 +MAX_CALLS = 1_000_000 + +# below this the median's confidence interval degenerates to the full range of +# what was seen, which is not a confidence interval. a table built from three +# rounds is how this suite published 0.2x for something that was 0.9x +MIN_ROUNDS = 9 + +# the control bounds the noise *within* a run: same process, same moment, same +# memory layout. it does not bound the drift *between* two runs, which is larger +# and which is what a baseline comparison is actually up against — two runs of +# an unchanged compiler, forty minutes apart on a 16-core laptop, disagreed by +# 8.4% on `words` while each reported a ±0.8% floor for itself. so the default +# bar for calling a change a change is set from that measurement rather than +# from the within-run floor. tighten it on a machine that does nothing else +BETWEEN_RUN_DRIFT = 0.10 + + +class Failure(Exception): + """something the run cannot honestly continue past""" + + +def load() -> str: + """what else the machine is doing, recorded at both ends of the run + + it does not change any decision the harness makes — the noise floor does + that, and it measures the effect rather than guessing at it. this is here so + that a table someone kept can be read back with the reason it was wide + """ + try: + return f"{os.getloadavg()[0]:.2f}" + except (OSError, AttributeError): + return "unknown" + + +# ── statistics ─────────────────────────────────────────────────────────────── +# +# the median throughout, never the mean and never the minimum. the mean is moved +# by the one round that hit a scheduler; the minimum is an extreme-value +# statistic whose expectation depends on how many rounds were run and on how +# quiet the machine happened to be, so it is not comparable between runs — which +# is the only thing this suite is for. + + +def median_interval(values: list[float], alpha: float = 0.05) -> tuple[float, float]: + """a distribution-free confidence interval for the median + + the order statistics either side of the middle, chosen from the binomial + tail. exact, assumes nothing about the shape of the noise, needs no + resampling and no random numbers — so two runs over the same data agree + """ + ordered = sorted(values) + n = len(ordered) + if n < 6: + return ordered[0], ordered[-1] + cumulative, k = 0.0, 0 + for i in range(n + 1): + p = math.comb(n, i) / 2**n + if cumulative + p > alpha / 2: + break + cumulative += p + k = i + 1 + if k == 0: + return ordered[0], ordered[-1] + return ordered[k - 1], ordered[n - k] + + +@dataclass +class Ratio: + """a paired ratio and how sure of it the run is""" + + median: float + low: float + high: float + + @property + def spread(self) -> float: + """half the interval, as a fraction of the median""" + return (self.high - self.low) / 2 / self.median if self.median else 0.0 + + def render(self, places: int = 2) -> str: + return f"{self.median:.{places}f}x ±{self.spread * 100:.1f}%" + + def as_json(self) -> dict: + return {"median": self.median, "low": self.low, "high": self.high} + + +def paired(numerator: list[float], denominator: list[float]) -> Ratio: + """the ratio of two builds, round by round rather than time against time + + dividing one build's median by another's would let a load spike that only + landed on one of them through unchallenged. these two lists are the same + rounds, so the spike is in both quotients and cancels + """ + ratios = [a / b for a, b in zip(numerator, denominator, strict=True)] + low, high = median_interval(ratios) + return Ratio(statistics.median(ratios), low, high) + + +# ── the manifest ───────────────────────────────────────────────────────────── + + +@dataclass +class Program: + name: str + group: str + measures: str + declines: int + mypyc: bool + + +def load_manifest(selected: list[str]) -> list[Program]: + """read `programs.toml`, and refuse to run against a corpus that has drifted + + the two lists — the files on disk and the entries in the manifest — are + meant to agree exactly. a file with no entry has no declared decline count + and so is unguarded; an entry with no file quietly measures nothing + """ + entries = tomllib.loads(MANIFEST.read_text())["programs"] + on_disk = {path.stem for path in PROGRAMS.glob("*.py")} + declared = set(entries) + + if undeclared := on_disk - declared: + raise Failure( + f"programs with no manifest entry: {', '.join(sorted(undeclared))}" + ) + if missing := declared - on_disk: + raise Failure(f"manifest entries with no program: {', '.join(sorted(missing))}") + + if selected: + if unknown := set(selected) - declared: + raise Failure(f"no such benchmark: {', '.join(sorted(unknown))}") + wanted = [name for name in entries if name in set(selected)] + else: + wanted = list(entries) + + if not wanted: + raise Failure("nothing selected") + + return [ + Program( + name=name, + group=entries[name]["group"], + measures=entries[name]["measures"], + declines=entries[name]["declines"], + mypyc=entries[name].get("mypyc", True), + ) + for name in wanted + ] + + +# ── staging and building ───────────────────────────────────────────────────── + + +@dataclass +class Leg: + """one build of one benchmark, and everything known about it""" + + name: str + module: str + compiled: bool + directory: Path + built: bool = False + error: str | None = None + log: Path | None = None + declines: list[str] = field(default_factory=list) + + def spec(self) -> dict: + return { + "name": self.name, + "module": self.module, + "compiled": self.compiled, + "dir": str(self.directory), + } + + +def stage(root: Path, program: Program, leg: str, python_version: str) -> Path: + """lay one build out as a project of its own, under a name of its own + + the name is what makes the whole method possible: four builds of the same + benchmark have to coexist in one process to be timed against each other, and + an extension module's init hook is found by its name, so they cannot share + one + """ + directory = root / program.name / leg + (directory / "dist").mkdir(parents=True) + module = f"{program.name}_{leg}" + shutil.copy(PROGRAMS / f"{program.name}.py", directory / f"{module}.py") + (directory / "pyproject.toml").write_text( + f'[project]\nname = "bench"\nversion = "0"\n' + f'requires-python = ">={python_version}"\n\n' + # a `float` annotation admits an `int` under python's numeric promotion, + # so without this every float place holds `int | float` and cannot be + # unboxed. mypyc reads a `float` annotation as a machine double, so this + # is what makes the two comparable at all — and it is recorded in the + # run's metadata because it changes the numbers + f"[tool.ty.analysis]\nstrict-float = true\n\n" + f'[tool.ty.environment]\npython-version = "{python_version}"\n' + ) + return directory + + +def build_by( + by: Path, directory: Path, module: str, python: str +) -> tuple[bool, str | None, list[str]]: + """compile one build, and read back what it refused to compile + + the decline list comes from the `--annotate` report rather than from the + diagnostics: the diagnostic renderer wraps and truncates, and a count taken + from it has been wrong before + """ + log = directory / "build.log" + result = subprocess.run( + [str(by), "compile", f"{module}.py", "-o", "out", "--annotate"], + cwd=directory, + env={**os.environ, "PYTHON": python}, + capture_output=True, + text=True, + ) + log.write_text(result.stdout + result.stderr) + if result.returncode != 0: + return False, f"`by compile` exited {result.returncode}", [] + + declines = [] + for report in (directory / "out").glob("*.annotated"): + section = report.read_text().partition("## left to the interpreted definition")[ + 2 + ] + for line in section.partition("\n##")[0].splitlines(): + if line.startswith("- "): + declines.append(line[2:]) + + artefacts = [ + p + for p in (directory / "out").iterdir() + if p.suffix in {".so", ".pyd", ".dylib"} + ] + if not artefacts: + return False, "`by compile` succeeded but left no extension module", declines + for artefact in artefacts: + shutil.copy(artefact, directory / "dist" / artefact.name) + return True, None, declines + + +def build_mypyc(directory: Path, module: str, python: str) -> tuple[bool, str | None]: + """compile one build with mypyc, and say so out loud when it will not + + the previous harness sent this to /dev/null, and a run in which mypyc was + simply broken was read as mypyc being unable to compile the program + """ + log = directory / "build.log" + result = subprocess.run( + [ + "uv", + "run", + "--no-project", + "--with", + "mypy", + "--with", + "setuptools", + "--python", + python, + "mypyc", + f"{module}.py", + ], + cwd=directory, + capture_output=True, + text=True, + ) + log.write_text(result.stdout + result.stderr) + artefacts = [ + p for p in directory.iterdir() if p.suffix in {".so", ".pyd", ".dylib"} + ] + if not artefacts: + return ( + False, + f"mypyc left no extension module (exit {result.returncode}); log at {log}", + ) + for artefact in artefacts: + shutil.copy(artefact, directory / "dist" / artefact.name) + return True, None + + +# ── driving the timer ──────────────────────────────────────────────────────── + + +def drive(python: str, spec: dict, work: Path, tag: str) -> dict: + path = work / f"spec-{tag}.json" + path.write_text(json.dumps(spec)) + result = subprocess.run( + [python, str(HERE / "timer.py"), str(path)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise Failure( + f"the timer failed for {tag} (exit {result.returncode}):\n{result.stderr}" + ) + return json.loads(result.stdout) + + +# ── the run ────────────────────────────────────────────────────────────────── + + +@dataclass +class Result: + program: Program + status: str + legs: dict[str, Leg] + times: dict[str, list[float]] = field(default_factory=dict) + calls: int = 0 + notes: list[str] = field(default_factory=list) + + def median(self, leg: str) -> float | None: + values = self.times.get(leg) + return statistics.median(values) if values else None + + def ratio(self, leg: str) -> Ratio | None: + if leg in self.times and "by" in self.times: + return paired(self.times[leg], self.times["by"]) + return None + + @property + def noise(self) -> float | None: + """what the suite reported for two builds that are the same program + + both how far the control landed from 1.00 and how far it wandered: a + build that reads consistently 2% away from its own twin is as much a + problem as one that reads all over the place + """ + control = self.ratio("control") + if control is None: + return None + return max(abs(control.median - 1), control.spread) + + def noisy(self, limit: float) -> bool: + return self.noise is not None and self.noise > limit + + +def run_program( + args, + program: Program, + root: Path, + python: str, + python_version: str, + built_after: float, +) -> Result: + legs: dict[str, Leg] = {} + + directory = stage(root, program, "cpython", python_version) + shutil.copy( + directory / f"{program.name}_cpython.py", + directory / "dist" / f"{program.name}_cpython.py", + ) + legs["cpython"] = Leg( + "cpython", f"{program.name}_cpython", False, directory / "dist", built=True + ) + + # `control` is a second, independent compile of the same source. it is the + # null: two builds of one program are one program, so whatever ratio the + # suite reports between them is noise it invented. that number is printed on + # every row, and nothing smaller than it is a finding + for leg in ("by", "control"): + directory = stage(root, program, leg, python_version) + module = f"{program.name}_{leg}" + built, error, declines = build_by(args.by, directory, module, python) + legs[leg] = Leg( + leg, + module, + True, + directory / "dist", + built, + error, + directory / "build.log", + declines, + ) + + if program.mypyc and not args.no_mypyc: + directory = stage(root, program, "mypyc", python_version) + module = f"{program.name}_mypyc" + built, error = build_mypyc(directory, module, python) + legs["mypyc"] = Leg( + "mypyc", + module, + True, + directory / "dist", + built, + error, + directory / "build.log", + ) + + result = Result(program, "ok", legs) + + # a decline that nobody declared means the row below is not measuring what + # its name says. exact match in both directions: a compiler that started + # compiling something is as much a change as one that stopped + found = len(legs["by"].declines) + if legs["by"].built and found != program.declines: + result.status = "declines" + result.notes.append( + f"expected {program.declines} declined function(s), found {found}" + + (": " + "; ".join(legs["by"].declines) if legs["by"].declines else "") + ) + return result + + for leg in legs.values(): + if not leg.built: + if leg.name in ("by", "control"): + result.status = "build" + result.notes.append(f"{leg.name}: {leg.error}") + return result + result.notes.append(f"{leg.name}: {leg.error}") + + live = [leg for leg in legs.values() if leg.built] + base = { + "root": str(root), + "built_after": built_after, + "legs": [leg.spec() for leg in live], + } + + probe = drive(python, {**base, "mode": "probe"}, root, f"{program.name}-probe") + for name, why in probe["refused"].items(): + if name in ("by", "control", "cpython"): + result.status = "refused" + result.notes.append(f"{name}: {why}") + return result + result.notes.append(f"{name}: {why}") + live = [leg for leg in live if leg.name != name] + + # a build that got a different answer is not a faster build. `bigint` leans + # on this: a compiler that wrapped at 64 bits fails here rather than posting + # a very good time + expected = probe["answers"]["cpython"] + for name, answer in probe["answers"].items(): + if answer != expected: + result.status = "disagrees" + result.notes.append( + f"{name} answered {answer}, cpython answered {expected}" + ) + return result + + if args.verify_only: + return result + + base["legs"] = [leg.spec() for leg in live] + calibration = drive( + python, + { + **base, + "mode": "calibrate", + "probe_target": PROBE_SAMPLE, + "min_sample": MIN_SAMPLE, + "max_sample": MAX_SAMPLE, + "max_count": MAX_CALLS, + }, + root, + f"{program.name}-calibrate", + ) + result.calls = calibration["count"] + + timed = drive( + python, + { + **base, + "mode": "time", + "count": result.calls, + "rounds": args.rounds, + "warmup": args.warmup, + }, + root, + f"{program.name}-time", + ) + result.times = timed["timings"] + return result + + +# ── reporting ──────────────────────────────────────────────────────────────── + + +def render( + results: list[Result], metadata: dict, show_declines: bool, limit: float +) -> None: + header = ( + f"{'benchmark':<15}{'group':<10}{'cpython':>10}{'by':>10}{'mypyc':>10}" + f" {'vs cpython':>15}{'vs mypyc':>16}{'noise':>10}{'dec':>5}" + ) + print() + print(f"python {metadata['python']} ({metadata['implementation']})") + print(f"by {metadata['by_version']} from {metadata['git']}") + print(f"mypy {metadata['mypy'] or 'unavailable'}") + print( + f"host {metadata['host']}, {metadata['cpus']} cpus, load {metadata['load_before']}" + ) + print( + f"method {metadata['rounds']} rounds after {metadata['warmup']} warmup, " + f"paired medians, strict-float on" + ) + print() + print(header) + print("-" * len(header)) + + for result in results: + name, group = result.program.name, result.program.group + if result.status != "ok": + print( + f"{name:<15}{group:<10} {result.status.upper()}: {'; '.join(result.notes)}" + ) + continue + cpython, by = result.median("cpython"), result.median("by") + mypyc = result.median("mypyc") + against_cpython = result.ratio("cpython") + against_mypyc = result.ratio("mypyc") + noise = result.noise + print( + f"{name:<15}{group:<10}" + f"{cpython * 1000:>9.2f}m{by * 1000:>9.2f}m" + f"{(f'{mypyc * 1000:.2f}m' if mypyc else '-'):>10}" + f" {(against_cpython.render() if against_cpython else '-'):>15}" + f"{(against_mypyc.render() if against_mypyc else '-'):>16}" + f"{f'±{noise * 100:.1f}%' + ('!' if result.noisy(limit) else ''):>10}" + f"{len(result.legs['by'].declines):>5}" + ) + for note in result.notes: + print(f"{'':<25}note: {note}") + + print() + floors = [r.noise for r in results if r.status == "ok" and r.noise is not None] + if floors: + print( + f"noise floor: median ±{statistics.median(floors) * 100:.2f}%, " + f"worst ±{max(floors) * 100:.2f}% — a change smaller than a row's own " + f"floor is not a change" + ) + failed = [r for r in results if r.status != "ok"] + noisy = [r for r in results if r.status == "ok" and r.noisy(limit)] + print( + f"{len(results) - len(failed) - len(noisy)}/{len(results)} benchmarks measured" + ) + if noisy: + # the row's two identical builds disagreed by more than the limit, so + # whatever else it says, it is not a measurement of anything. marked + # rather than deleted: the numbers are still worth a glance, and the + # `!` is there to stop one being quoted + print( + f"{len(noisy)} too noisy to trust (marked `!`): " + f"{', '.join(r.program.name for r in noisy)} — rerun on a quieter machine" + ) + + if show_declines: + for result in results: + if result.legs.get("by") and result.legs["by"].declines: + print(f"\n{result.program.name} declines:") + for decline in result.legs["by"].declines: + print(f" {decline}") + + +def render_verification(results: list[Result]) -> None: + """everything the suite checks that does not involve a clock + + this half is deterministic, so it is the half that can run anywhere — a + shared runner cannot time anything, but it can tell you that a benchmark + stopped compiling, or that the compiled build started answering differently + """ + print() + for result in results: + declines = len(result.legs["by"].declines) if "by" in result.legs else 0 + mark = "ok " if result.status == "ok" else result.status.upper() + print( + f"{mark:<10}{result.program.name:<15}{declines:>3} declined" + + (f" {'; '.join(result.notes)}" if result.notes else "") + ) + failed = [r for r in results if r.status != "ok"] + print(f"\n{len(results) - len(failed)}/{len(results)} verified") + + +def compare( + results: list[Result], baseline: dict, threshold: float, limit: float +) -> bool: + """a regression is detected rather than eyeballed + + the quantity compared between runs is the speedup, not the time: absolute + times are not comparable across machines or across a Tuesday, and this suite + exists because they were compared anyway + """ + print("\nagainst the baseline") + for key in ("python", "implementation", "host"): + if baseline["metadata"].get(key) != CURRENT[key]: + print( + f" warning: {key} was {baseline['metadata'].get(key)!r} then and is " + f"{CURRENT[key]!r} now — the two runs are not comparable" + ) + + print(f" {'benchmark':<15}{'then':>12}{'now':>12}{'change':>12} verdict") + regressed = False + for result in results: + previous = baseline["benchmarks"].get(result.program.name) + if result.status != "ok" or previous is None or previous.get("status") != "ok": + continue + old = previous["vs_cpython"]["median"] + new = result.ratio("cpython").median + change = new / old - 1 + old_noise, new_noise = previous["noise"], result.noise + # a row either run could not measure is skipped rather than compared + # with a very wide bar. a pair of identical builds that disagreed by 70% + # says the machine was preempting samples, and under preemption the + # longer sample loses more — so the paired ratio does not merely get + # noisier, it drifts upwards. widening the bar does not fix a bias + if max(old_noise, new_noise) > limit: + print( + f" {result.program.name:<15}{old:>11.2f}x{new:>11.2f}x" + f"{change * 100:>11.1f}% skipped: ±{max(old_noise, new_noise) * 100:.0f}% " + f"noise in one of the two runs" + ) + continue + # the bar is whichever is larger: the noise the two runs measured for + # themselves, or the floor asked for on the command line + bar = max(old_noise + new_noise, threshold) + if change < -bar: + verdict, regressed = "REGRESSED", True + elif change > bar: + verdict = "improved" + else: + verdict = f"same (within ±{bar * 100:.1f}%)" + print( + f" {result.program.name:<15}{old:>11.2f}x{new:>11.2f}x" + f"{change * 100:>11.1f}% {verdict}" + ) + return regressed + + +CURRENT: dict = {} + + +# ── the harness's check on itself ──────────────────────────────────────────── + + +def self_check(by: Path, python: str, python_version: str) -> int: + """build each way this suite has lied before, and prove it is refused now + + an ablation harness here once matched nothing and reported that every edit + cost nothing, and the reason it was believed is that a harness which cannot + fail looks exactly like a harness that found nothing wrong. so the refusals + are exercised rather than asserted, by handing the timer a leg that is + wrong in each of the ways a leg has actually been wrong + """ + root = Path(tempfile.mkdtemp(prefix="native-bench-selfcheck-")) + failures = [] + + def expect(what: str, refusal: str | None, wanted: str) -> None: + if refusal is None: + failures.append(f"{what}: was accepted, and should have been refused") + elif wanted not in refusal: + failures.append(f"{what}: refused with {refusal!r}, wanted {wanted!r}") + else: + print(f" refused {what}: {refusal}") + + def refusal_for(leg: dict, built_after: float = 0.0) -> str | None: + answer = drive( + python, + { + "mode": "probe", + "root": str(root), + "built_after": built_after, + "legs": [leg], + }, + root, + "selfcheck", + ) + return answer["refused"].get(leg["name"]) + + # a real extension module to be wrong about + stage = root / "real" + (stage / "dist").mkdir(parents=True) + (stage / "canary.py").write_text("def bench() -> int:\n return 1\n") + (stage / "pyproject.toml").write_text( + f'[project]\nname = "c"\nversion = "0"\nrequires-python = ">={python_version}"\n' + ) + built, error, _ = build_by(by, stage, "canary", python) + if not built: + print( + f"error: the self-check could not build its own canary: {error}", + file=sys.stderr, + ) + return 2 + + print("the guards:") + expect( + "an interpreted build passed off as a compiled one", + refusal_for( + {"name": "by", "module": "canary", "compiled": True, "dir": str(stage)} + ), + "not an extension module", + ) + expect( + "a compiled build passed off as an interpreted one", + refusal_for( + { + "name": "cpython", + "module": "canary", + "compiled": False, + "dir": str(stage / "dist"), + } + ), + "is an extension module", + ) + # the stale-output-directory failure: the build did not happen, and the + # artefact left over from last time answered in its place + expect( + "an artefact older than this run's build", + refusal_for( + { + "name": "by", + "module": "canary", + "compiled": True, + "dir": str(stage / "dist"), + }, + built_after=os.stat(stage).st_mtime + 3600, + ), + "predates this run's build", + ) + outside = Path(tempfile.mkdtemp(prefix="native-bench-outside-")) + for artefact in (stage / "dist").glob("canary*"): + shutil.copy(artefact, outside / artefact.name) + expect( + "an artefact from outside this run's root", + refusal_for( + {"name": "by", "module": "canary", "compiled": True, "dir": str(outside)} + ), + "outside this run's root", + ) + expect( + "a build that is not there at all", + refusal_for( + { + "name": "by", + "module": "absent", + "compiled": True, + "dir": str(stage / "dist"), + } + ), + "import failed", + ) + + # and the corpus guards, which are what stops a run measuring nothing + for what, selection, wanted in ( + ("an unknown benchmark name", ["nosuchbenchmark"], "no such benchmark"), + ): + try: + load_manifest(selection) + failures.append(f"{what}: was accepted") + except Failure as failure: + if wanted not in str(failure): + failures.append(f"{what}: refused with {failure!r}") + else: + print(f" refused {what}: {failure}") + + shutil.rmtree(root, ignore_errors=True) + shutil.rmtree(outside, ignore_errors=True) + for failure in failures: + print(f"error: {failure}", file=sys.stderr) + print( + "\nevery guard held" + if not failures + else f"\n{len(failures)} guard(s) did not hold" + ) + return 1 if failures else 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "programs", nargs="*", help="benchmarks to run (default: all of them)" + ) + parser.add_argument( + "--by", + type=Path, + default=ROOT / "target" / "release" / "by", + help="the compiler to measure (default: this checkout's release build)", + ) + parser.add_argument( + "--python", + default="3.13", + help="interpreter to build against and run: a version for `uv python find`, or a path", + ) + parser.add_argument( + "--rounds", + type=int, + default=21, + help="timed rounds per benchmark (default: 21)", + ) + parser.add_argument( + "--warmup", type=int, default=3, help="untimed rounds first (default: 3)" + ) + parser.add_argument("--no-mypyc", action="store_true", help="skip the mypyc build") + parser.add_argument( + "--declines", action="store_true", help="list every declined function" + ) + parser.add_argument( + "--json", type=Path, help="write the full result for a later comparison" + ) + parser.add_argument( + "--baseline", type=Path, help="compare against a previous --json" + ) + parser.add_argument( + "--regression-threshold", + type=float, + default=BETWEEN_RUN_DRIFT, + help=f"the smallest change worth calling one, when noise is smaller " + f"(default: {BETWEEN_RUN_DRIFT})", + ) + parser.add_argument( + "--noise-limit", + type=float, + default=0.10, + help="the noise a row may show and still count as measured (default: 0.10)", + ) + parser.add_argument( + "--keep", action="store_true", help="keep the build tree and say where it is" + ) + parser.add_argument( + "--self-check", + action="store_true", + help="prove the guards refuse what they are meant to, and run nothing else", + ) + parser.add_argument( + "--verify-only", + action="store_true", + help="build, prove and check declines, but time nothing (deterministic, so it runs anywhere)", + ) + args = parser.parse_args() + + if args.rounds < MIN_ROUNDS: + raise Failure( + f"--rounds {args.rounds} is not a measurement: below {MIN_ROUNDS} rounds the " + "confidence interval degenerates to the range of what was seen" + ) + + if not args.self_check: + programs = load_manifest(args.programs) + + if not args.by.is_file() or not os.access(args.by, os.X_OK): + raise Failure( + f"no compiler at {args.by} — build one with `cargo build --release --bin by`.\n" + "a debug build is far too slow to measure anything with" + ) + + if Path(args.python).is_absolute(): + python = args.python + else: + found = subprocess.run( + ["uv", "python", "find", args.python], capture_output=True, text=True + ) + if found.returncode != 0: + raise Failure(f"no interpreter for {args.python!r}: {found.stderr.strip()}") + python = found.stdout.strip() + + probe = subprocess.run( + [ + python, + "-c", + "import platform,sys;" + "print(platform.python_version());print(platform.python_implementation());" + "print(sys.version.split()[0])", + ], + capture_output=True, + text=True, + ) + if probe.returncode != 0: + raise Failure(f"{python} does not run: {probe.stderr.strip()}") + full_version, implementation, _ = probe.stdout.split() + python_version = ".".join(full_version.split(".")[:2]) + + if args.self_check: + return self_check(args.by, python, python_version) + + version = subprocess.run( + [str(args.by), "--version"], capture_output=True, text=True + ).stdout.strip() + git = subprocess.run( + ["git", "-C", str(ROOT), "describe", "--always", "--dirty"], + capture_output=True, + text=True, + ).stdout.strip() + mypy = "skipped" if args.no_mypyc else None + if not args.no_mypyc: + found = subprocess.run( + [ + "uv", + "run", + "--no-project", + "--with", + "mypy", + "--python", + python, + "mypy", + "--version", + ], + capture_output=True, + text=True, + ) + mypy = found.stdout.strip() if found.returncode == 0 else None + if mypy is None: + print( + f"warning: mypyc is unavailable, so nothing will be compared against it\n{found.stderr}", + file=sys.stderr, + ) + + CURRENT.update( + { + "python": full_version, + "implementation": implementation, + "by_version": version, + "git": git or "unknown", + "mypy": mypy, + "host": f"{platform.system()} {platform.machine()}", + "cpus": os.cpu_count(), + "load_before": load(), + "rounds": args.rounds, + "warmup": args.warmup, + "strict_float": True, + } + ) + + # a fresh root every run, so a stale artefact cannot be picked up by + # construction rather than by remembering to delete one + root = Path(tempfile.mkdtemp(prefix="native-bench-")) + built_after = root.stat().st_mtime + + results = [] + for index, program in enumerate(programs, 1): + print(f"[{index}/{len(programs)}] {program.name}", file=sys.stderr) + results.append( + run_program(args, program, root, python, python_version, built_after) + ) + + CURRENT["load_after"] = load() + CURRENT["noise_limit"] = args.noise_limit + if args.verify_only: + render_verification(results) + if args.keep: + print(f"build tree kept at {root}") + else: + shutil.rmtree(root, ignore_errors=True) + failed = [r for r in results if r.status != "ok"] + return 1 if failed else 0 + render(results, CURRENT, args.declines, args.noise_limit) + + payload = { + "metadata": CURRENT, + "benchmarks": { + result.program.name: { + "status": result.status, + "group": result.program.group, + "notes": result.notes, + "calls": result.calls, + "declines": result.legs["by"].declines if "by" in result.legs else [], + "times": result.times, + **( + { + "vs_cpython": result.ratio("cpython").as_json(), + "vs_mypyc": result.ratio("mypyc").as_json() + if result.ratio("mypyc") + else None, + "control": result.ratio("control").as_json(), + "noise": result.noise, + "noisy": result.noisy(args.noise_limit), + } + if result.status == "ok" + else {} + ), + } + for result in results + }, + } + if args.json: + args.json.write_text(json.dumps(payload, indent=2)) + print(f"written to {args.json}") + + regressed = False + if args.baseline: + regressed = compare( + results, + json.loads(args.baseline.read_text()), + args.regression_threshold, + args.noise_limit, + ) + + if args.keep: + print(f"build tree kept at {root}") + else: + shutil.rmtree(root, ignore_errors=True) + + failed = [r for r in results if r.status != "ok"] + noisy = [r for r in results if r.status == "ok" and r.noisy(args.noise_limit)] + if failed: + print( + f"\n{len(failed)} benchmark(s) did not measure: " + f"{', '.join(r.program.name for r in failed)}", + file=sys.stderr, + ) + # a run with a junk row in it exits non-zero even when every row it *could* + # measure looks fine. the alternative is a table that is mostly trustworthy, + # which is the kind nobody remembers to check before quoting + return 1 if failed or noisy or regressed else 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as failure: + print(f"error: {failure}", file=sys.stderr) + raise SystemExit(2) from None diff --git a/scripts/native-bench/programs.toml b/scripts/native-bench/programs.toml new file mode 100644 index 0000000000..580657f3fa --- /dev/null +++ b/scripts/native-bench/programs.toml @@ -0,0 +1,150 @@ +# the benchmark corpus, and the ledger of what the native backend refuses +# +# `group` is the axis a benchmark exists to measure. `measures` is the one-line +# reason it is in the set — if two entries would have the same one, one of them +# should go. +# +# `declines` is how many functions `by compile` currently leaves to their +# interpreted definitions. it is checked exactly: a benchmark whose count moved +# in either direction fails the run rather than quietly reporting a number about +# interpreted code. improve the compiler and this file changes in the same +# commit, which makes it a readable history of what the backend learned to take. +# +# `mypyc = false` means mypyc genuinely cannot build the program — the harness +# reports a *failed* mypyc build loudly, so this is for the cases that are not +# failures, and nothing may be listed here without the log to justify it. + +[programs.mandel] +group = "float" +measures = "scalar float arithmetic with a call in the inner loop" +declines = 0 + +[programs.mandel_inline] +group = "float" +measures = "the same work with the loop spelled out — the pair isolates call cost" +declines = 0 + +[programs.loops] +group = "int" +measures = "tagged-integer arithmetic with no container, no call and no float" +declines = 0 + +[programs.bigint] +group = "int" +measures = "integer arithmetic that leaves the machine word" +declines = 0 + +[programs.calls] +group = "dispatch" +measures = "a plain function call with a one-add body" +declines = 0 + +[programs.methods] +group = "dispatch" +measures = "a method call on a long-lived object" +declines = 0 + +[programs.recurse] +group = "dispatch" +measures = "calls that nest rather than repeat" +declines = 0 + +[programs.inherit] +group = "dispatch" +measures = "an override reached through a base-typed local" +declines = 0 + +[programs.alloc] +group = "memory" +measures = "an object built and dropped every iteration" +declines = 0 + +[programs.fields] +group = "memory" +measures = "attribute reads and writes on one long-lived object" +declines = 0 + +[programs.objects] +group = "memory" +measures = "allocation, fields and methods together, as object code looks" +declines = 0 + +[programs.dot] +group = "list" +measures = "indexed reads over two lists that never escape" +declines = 0 + +[programs.prefix] +group = "list" +measures = "appends into a list the function owns, iterated with `for`" +declines = 0 + +[programs.sieve] +group = "list" +measures = "indexed writes into a `bool` buffer from a nested loop" +declines = 0 + +[programs.tuples] +group = "tuple" +measures = "tuples packed, unpacked and indexed, and a two-value return" +declines = 0 + +[programs.sets] +group = "set" +measures = "a set built once and tested for membership many times" +declines = 0 + +[programs.dictget] +group = "dict" +measures = "dict lookup and in-place update, keys handed in already built" +declines = 0 + +[programs.dicthist] +group = "dict" +measures = "the histogram shape: membership test then read-modify-write" +declines = 0 + +[programs.words] +group = "str" +measures = "string concatenation in a loop" +declines = 0 + +[programs.chars] +group = "str" +measures = "an indexed scan over text, a character at a time" +declines = 0 + +[programs.strops] +group = "str" +measures = "the `str` methods real code calls: split, join, startswith, upper" +declines = 0 + +[programs.keybuild] +group = "str" +measures = "a builtin through the module namespace, a call, and a concatenation" +declines = 0 + +[programs.generic] +group = "boxing" +measures = "a list handed to a function with a type parameter" +declines = 0 + +[programs.generic_mono] +group = "boxing" +measures = "the same call monomorphised — the pair isolates what boxing costs" +declines = 0 + +[programs.excs] +group = "control" +measures = "a raise that is caught, and a `try` that never fires" +declines = 0 + +[programs.gen] +group = "frames" +measures = "a generator consumed by a `for` loop" +declines = 0 + +[programs.coro] +group = "frames" +measures = "awaits that complete immediately, driven without an event loop" +declines = 0 diff --git a/scripts/native-bench/programs/alloc.py b/scripts/native-bench/programs/alloc.py new file mode 100644 index 0000000000..6bcdd89d4f --- /dev/null +++ b/scripts/native-bench/programs/alloc.py @@ -0,0 +1,27 @@ +"""an object built and dropped every iteration, and nothing kept + +allocation, the field stores the constructor makes, and the deallocation that +follows when the last reference goes. `objects` allocates too but then calls +methods on what it built, so its number is a mixture; this one is the mixture's +allocation half on its own +""" + + +class Pair: + def __init__(self, x: int, y: int) -> None: + self.x = x + self.y = y + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + pair = Pair(i, i + 1) + total = total + pair.x + i = i + 1 + return total + + +def bench() -> int: + return run(300000) diff --git a/scripts/native-bench/programs/bigint.py b/scripts/native-bench/programs/bigint.py new file mode 100644 index 0000000000..6e85247fb1 --- /dev/null +++ b/scripts/native-bench/programs/bigint.py @@ -0,0 +1,33 @@ +"""integer arithmetic that leaves the machine word + +a compiler that unboxes `int` has to decide what happens when the value stops +fitting. the answer is checked as well as timed: `bench()` returns a number far +wider than 64 bits, so a build that wrapped instead of promoting fails the +agreement check rather than posting a fast time +""" + + +def factorial(n: int) -> int: + out = 1 + i = 2 + while i <= n: + out = out * i + i = i + 1 + return out + + +def digits(value: int) -> int: + count = 0 + while value > 0: + value = value // 10 + count = count + 1 + return count + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + total = total + digits(factorial(300)) + r = r + 1 + return total + factorial(60) diff --git a/scripts/native-bench/programs/calls.py b/scripts/native-bench/programs/calls.py new file mode 100644 index 0000000000..ab749e196a --- /dev/null +++ b/scripts/native-bench/programs/calls.py @@ -0,0 +1,24 @@ +"""a plain function call, with as little else in the loop as it can have + +the body is one add, so what the loop measures is the calling convention: the +frame, the argument passing, and the error check after the return. `mandel` has +a call on its hot path too but pays for float arithmetic around it, which is the +thing that hid a call regression once already +""" + + +def add(a: int, b: int) -> int: + return a + b + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = add(total, i) + i = i + 1 + return total + + +def bench() -> int: + return run(400000) diff --git a/scripts/native-bench/programs/chars.py b/scripts/native-bench/programs/chars.py new file mode 100644 index 0000000000..87a5ab9600 --- /dev/null +++ b/scripts/native-bench/programs/chars.py @@ -0,0 +1,37 @@ +"""a scan over text, a character at a time: indexing and comparison + +the text is made by repetition rather than by concatenation so that `words` — +which is the concatenation benchmark — cannot leak into this measurement +""" + + +def text(n: int) -> str: + unit = "word0 word1 word2 word3 word4 word5 word6 word7 word8 word9 " + return unit * n + + +def longest_run(line: str) -> int: + best = 0 + run = 0 + i = 0 + while i < len(line): + if line[i] == " ": + if run > best: + best = run + run = 0 + else: + run = run + 1 + i = i + 1 + if run > best: + best = run + return best + + +def bench() -> int: + line = text(2000) + total = 0 + r = 0 + while r < 10: + total = total + longest_run(line) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/coro.py b/scripts/native-bench/programs/coro.py new file mode 100644 index 0000000000..8b6dc730fe --- /dev/null +++ b/scripts/native-bench/programs/coro.py @@ -0,0 +1,40 @@ +"""awaits that complete immediately, driven without an event loop + +the loop is driven by hand rather than through `asyncio` on purpose: an event +loop would swamp the thing being measured with scheduling, and most awaits in +real code complete without ever suspending. what this times is the frame — its +creation, its resume, and the await of a coroutine that is already done +""" + + +async def step(i: int) -> int: + return (i * 7) % 13 + + +async def chain(n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + await step(i) + i = i + 1 + return total + + +def drive(n: int) -> int: + """run a coroutine to completion, with no loop underneath it""" + coroutine = chain(n) + try: + coroutine.send(None) + except StopIteration as done: + return done.value + coroutine.close() + return -1 + + +def bench() -> int: + total = 0 + run = 0 + while run < 20: + total = total + drive(2000) + run = run + 1 + return total diff --git a/scripts/native-bench/programs/dictget.py b/scripts/native-bench/programs/dictget.py new file mode 100644 index 0000000000..97c8e0b109 --- /dev/null +++ b/scripts/native-bench/programs/dictget.py @@ -0,0 +1,33 @@ +"""dict lookup and in-place update, with the keys handed in already built + +the keys are built once and the timed loop only subscripts, so this is the +lookup question and nothing else. `str` keys, because that is what a real table +is keyed by and where cpython's own lookup is most specialised +""" + + +def total(table: dict[str, int], keys: list[str], passes: int) -> int: + running = 0 + p = 0 + n = len(keys) + while p < passes: + i = 0 + while i < n: + key = keys[i] + running = running + table[key] + table[key] = table[key] + 1 + i = i + 1 + p = p + 1 + return running + + +def bench() -> int: + keys = [] + table: dict[str, int] = {} + i = 0 + while i < 2000: + key = "k" + str(i) + keys.append(key) + table[key] = i + i = i + 1 + return total(table, keys, 50) diff --git a/scripts/native-bench/programs/dicthist.py b/scripts/native-bench/programs/dicthist.py new file mode 100644 index 0000000000..1c101f01d1 --- /dev/null +++ b/scripts/native-bench/programs/dicthist.py @@ -0,0 +1,34 @@ +"""the histogram shape: a membership test, then a read-modify-write on the hit + +the word list is built once and counted many times, so counting is what the +number is about. the miss on the first sighting of each word is the branch that +makes this shape its own, and is why it is not folded into `dictget` +""" + + +def counted(words: list[str]) -> int: + seen: dict[str, int] = {} + for word in words: + if word in seen: + seen[word] = seen[word] + 1 + else: + seen[word] = 1 + return len(seen) + + +def repeated(words: list[str], passes: int) -> int: + total = 0 + p = 0 + while p < passes: + total = total + counted(words) + p = p + 1 + return total + + +def bench() -> int: + words = [] + i = 0 + while i < 20000: + words.append("w" + str(i % 500)) + i = i + 1 + return repeated(words, 10) diff --git a/scripts/native-bench/programs/dot.py b/scripts/native-bench/programs/dot.py new file mode 100644 index 0000000000..2701183cf8 --- /dev/null +++ b/scripts/native-bench/programs/dot.py @@ -0,0 +1,31 @@ +"""indexed reads over two lists that never escape the function + +the list is the one container the compiler can hold as a buffer, so this is the +best case for it: no growth, no writes, and the element type known at the +definition +""" + + +def dot(a: list[float], b: list[float]) -> float: + out = 0.0 + i = 0 + while i < len(a): + out = out + a[i] * b[i] + i = i + 1 + return out + + +def bench() -> float: + a = [] + b = [] + i = 0 + while i < 50000: + a.append(i * 0.5) + b.append(i * 0.25) + i = i + 1 + total = 0.0 + r = 0 + while r < 10: + total = total + dot(a, b) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/excs.py b/scripts/native-bench/programs/excs.py new file mode 100644 index 0000000000..7c38d05181 --- /dev/null +++ b/scripts/native-bench/programs/excs.py @@ -0,0 +1,55 @@ +"""a raise that is caught, on the path the program expects to take + +every other benchmark stays on the straight line. this one leaves it: the +compiler emits an error edge from any fallible operation, and a `try` turns that +edge into a branch to a handler rather than to the function's own exit. + +two shapes, because they cost differently. `caught` raises and catches across a +call boundary, which is the expensive one. `guarded` never raises at all and +measures what merely being inside a `try` costs a loop that succeeds — which +should be nothing, and is worth knowing rather than assuming +""" + + +class Refused(Exception): + pass + + +def parse(text: str) -> int: + if len(text) == 0: + raise Refused("empty") + total = 0 + i = 0 + while i < len(text): + total = total + ord(text[i]) + i = i + 1 + return total + + +def caught(rounds: int) -> int: + total = 0 + i = 0 + while i < rounds: + try: + total = total + parse("" if i % 4 == 0 else "abc") + except Refused: + total = total + 1 + i = i + 1 + return total + + +def guarded(rounds: int) -> int: + """the same loop with a handler that is never reached""" + total = 0 + i = 0 + while i < rounds: + try: + total = total + parse("abc") + except Refused: + total = total + 1 + i = i + 1 + return total + + +def bench() -> int: + return caught(20000) + guarded(20000) diff --git a/scripts/native-bench/programs/fields.py b/scripts/native-bench/programs/fields.py new file mode 100644 index 0000000000..37c166e071 --- /dev/null +++ b/scripts/native-bench/programs/fields.py @@ -0,0 +1,27 @@ +"""attribute reads and writes on one object that outlives the loop + +nothing is allocated inside the timed loop and no method is called, so this is +the field access on its own: on an emitted class that is an offset into a fixed +layout, and on an interpreted one it is a dict lookup through `__dict__` +""" + + +class State: + def __init__(self) -> None: + self.a = 0 + self.b = 1 + self.c = 2 + + +def run(state: State, n: int) -> int: + i = 0 + while i < n: + state.a = state.b + state.c + state.b = state.a - state.c + state.c = state.c + 1 + i = i + 1 + return state.a + state.b + state.c + + +def bench() -> int: + return run(State(), 300000) diff --git a/scripts/native-bench/programs/gen.py b/scripts/native-bench/programs/gen.py new file mode 100644 index 0000000000..24ffd768f2 --- /dev/null +++ b/scripts/native-bench/programs/gen.py @@ -0,0 +1,30 @@ +"""a generator consumed by a `for` loop + +the compiler turns a resumable frame into a state object: locals parked in +fields, a tag saying whether a suspension was a `yield` or an `await`, and a +resume method. `coro` covers the await half of that; this is the yield half, and +it is also the half real code writes far more often +""" + + +def steps(n: int): + i = 0 + while i < n: + yield (i * 7) % 13 + i = i + 1 + + +def consume(n: int) -> int: + total = 0 + for value in steps(n): + total = total + value + return total + + +def bench() -> int: + total = 0 + r = 0 + while r < 20: + total = total + consume(2000) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/generic.py b/scripts/native-bench/programs/generic.py new file mode 100644 index 0000000000..ff007240ab --- /dev/null +++ b/scripts/native-bench/programs/generic.py @@ -0,0 +1,32 @@ +"""a list built in a hot loop and handed to a generic function + +the type parameter is what stops the list being a buffer: the element +representation is not known at the definition, so neither the callee's body nor +the caller's list can be unboxed. `generic_mono` is the same program with the +call monomorphised by hand, and the gap between the two is the whole measurement +""" + + +def consume[T](xs: list[T], ys: list[T]) -> int: + n = 0 + i = 0 + while i < len(xs): + n = n + 1 + i = i + 1 + return n + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + xs = [0.0] + ys = [0.0] + i = 0 + while i < 2000: + xs.append(i * 0.5) + ys.append(i * 0.25) + i = i + 1 + total = total + consume(xs, ys) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/generic_mono.py b/scripts/native-bench/programs/generic_mono.py new file mode 100644 index 0000000000..e2b1047875 --- /dev/null +++ b/scripts/native-bench/programs/generic_mono.py @@ -0,0 +1,31 @@ +"""`generic`, with the call monomorphised by hand + +identical in every other respect, on purpose: read the two rows as a pair and +the difference is what the type parameter costs. read either one alone and it +says almost nothing +""" + + +def consume(xs: list[float], ys: list[float]) -> int: + n = 0 + i = 0 + while i < len(xs): + n = n + 1 + i = i + 1 + return n + + +def bench() -> int: + total = 0 + r = 0 + while r < 40: + xs = [0.0] + ys = [0.0] + i = 0 + while i < 2000: + xs.append(i * 0.5) + ys.append(i * 0.25) + i = i + 1 + total = total + consume(xs, ys) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/inherit.py b/scripts/native-bench/programs/inherit.py new file mode 100644 index 0000000000..16bb3a936d --- /dev/null +++ b/scripts/native-bench/programs/inherit.py @@ -0,0 +1,44 @@ +"""an override reached through a base-typed local + +the call site cannot know which body it runs, so the direct call `methods` +measures is not available to it. every other object benchmark here has exactly +one candidate; this is the one that does not, which is the case a devirtualising +compiler has to get right and a direct-calling one has to get wrong loudly +""" + + +class Shape: + def __init__(self, size: int) -> None: + self.size = size + + def area(self) -> int: + return self.size + + +class Square(Shape): + def area(self) -> int: + return self.size * self.size + + +def total(shapes: list[Shape], passes: int) -> int: + running = 0 + p = 0 + while p < passes: + i = 0 + while i < len(shapes): + running = running + shapes[i].area() + i = i + 1 + p = p + 1 + return running + + +def bench() -> int: + shapes: list[Shape] = [] + i = 0 + while i < 200: + if i % 2 == 0: + shapes.append(Shape(i)) + else: + shapes.append(Square(i)) + i = i + 1 + return total(shapes, 300) diff --git a/scripts/native-bench/programs/keybuild.py b/scripts/native-bench/programs/keybuild.py new file mode 100644 index 0000000000..1efa72e6ef --- /dev/null +++ b/scripts/native-bench/programs/keybuild.py @@ -0,0 +1,22 @@ +"""`"k" + str(i)`: a builtin through the module namespace, a call, and a concat + +there is no dict here at all. this expression sat in every loop of the dict +benchmarks and was most of what they measured, so it is timed on its own and +they hand their keys in already built. `len` is called once, at the end +""" + + +def keys(n: int, passes: int) -> int: + last = "k" + p = 0 + while p < passes: + i = 0 + while i < n: + last = "k" + str(i) + i = i + 1 + p = p + 1 + return len(last) + + +def bench() -> int: + return keys(2000, 25) diff --git a/scripts/native-bench/programs/loops.py b/scripts/native-bench/programs/loops.py new file mode 100644 index 0000000000..cb6a7d0de4 --- /dev/null +++ b/scripts/native-bench/programs/loops.py @@ -0,0 +1,30 @@ +"""tagged-integer arithmetic with no container, no call and no float + +every other integer benchmark here carries something else: `sieve` owns a list, +`mandel` is floats, `recurse` is calls. this one is the arithmetic on its own, +so it is the floor a compiled loop can be measured against +""" + + +def collatz(n: int) -> int: + steps = 0 + while n != 1: + if n % 2 == 0: + n = n // 2 + else: + n = 3 * n + 1 + steps = steps + 1 + return steps + + +def total(limit: int) -> int: + running = 0 + i = 1 + while i < limit: + running = running + collatz(i) + i = i + 1 + return running + + +def bench() -> int: + return total(6000) diff --git a/scripts/native-bench/programs/mandel.py b/scripts/native-bench/programs/mandel.py new file mode 100644 index 0000000000..96eb6d39f6 --- /dev/null +++ b/scripts/native-bench/programs/mandel.py @@ -0,0 +1,38 @@ +"""scalar float arithmetic with a call in the inner loop + +the pair with `mandel_inline` is the measurement: same work, one with the escape +loop behind a call and one with it spelled out, so the difference is what a call +costs on the hot path +""" + + +def escape(cr: float, ci: float, limit: int) -> int: + zr = 0.0 + zi = 0.0 + k = 0 + while k < limit: + if zr * zr + zi * zi > 4.0: + return k + t = zr * zr - zi * zi + cr + zi = 2.0 * zr * zi + ci + zr = t + k = k + 1 + return limit + + +def render(width: int, height: int, limit: int) -> int: + total = 0 + y = 0 + while y < height: + x = 0 + while x < width: + cr = -2.0 + 3.0 * x / width + ci = -1.2 + 2.4 * y / height + total = total + escape(cr, ci, limit) + x = x + 1 + y = y + 1 + return total + + +def bench() -> int: + return render(120, 120, 40) diff --git a/scripts/native-bench/programs/mandel_inline.py b/scripts/native-bench/programs/mandel_inline.py new file mode 100644 index 0000000000..a1c006964d --- /dev/null +++ b/scripts/native-bench/programs/mandel_inline.py @@ -0,0 +1,35 @@ +"""the same work as `mandel`, with the escape loop inlined + +the shape a nested loop takes when nobody factored the inner one out: the outer +body's locals feed the inner arithmetic and the inner loop leaves by `break`. +this once made the type checker not converge at all, so it is a canary as well +as a benchmark +""" + + +def render(width: int, height: int, limit: int) -> int: + total = 0 + y = 0 + while y < height: + x = 0 + while x < width: + cr = -2.0 + 3.0 * x / width + ci = -1.2 + 2.4 * y / height + zr = 0.0 + zi = 0.0 + k = 0 + while k < limit: + if zr * zr + zi * zi > 4.0: + break + t = zr * zr - zi * zi + cr + zi = 2.0 * zr * zi + ci + zr = t + k = k + 1 + total = total + k + x = x + 1 + y = y + 1 + return total + + +def bench() -> int: + return render(120, 120, 40) diff --git a/scripts/native-bench/programs/methods.py b/scripts/native-bench/programs/methods.py new file mode 100644 index 0000000000..f1cd72e7f7 --- /dev/null +++ b/scripts/native-bench/programs/methods.py @@ -0,0 +1,28 @@ +"""a method call on a long-lived object, with the object built once + +the companion to `calls`: same trivial body, reached through an instance rather +than through a module global. the difference between the two is what the method +lookup costs, and neither of them allocates — `alloc` is where allocation is +measured +""" + + +class Counter: + def __init__(self, base: int) -> None: + self.base = base + + def step(self, k: int) -> int: + return self.base + k + + +def run(counter: Counter, n: int) -> int: + total = 0 + i = 0 + while i < n: + total = total + counter.step(i) + i = i + 1 + return total + + +def bench() -> int: + return run(Counter(3), 300000) diff --git a/scripts/native-bench/programs/objects.py b/scripts/native-bench/programs/objects.py new file mode 100644 index 0000000000..1ab45e52e3 --- /dev/null +++ b/scripts/native-bench/programs/objects.py @@ -0,0 +1,30 @@ +"""a short-lived object that is then used: allocation, fields and methods together + +deliberately the mixture, because that is what object-shaped code looks like. +`alloc`, `fields` and `methods` are its three parts measured apart, and a change +that moves this one without moving any of those is worth explaining +""" + + +class Vec: + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + def norm2(self) -> float: + return self.x * self.x + self.y * self.y + + def shift(self, dx: float, dy: float) -> float: + self.x = self.x + dx + self.y = self.y + dy + return self.x + self.y + + +def bench() -> float: + total = 0.0 + i = 0 + while i < 200000: + v = Vec(i * 0.5, i * 0.25) + total = total + v.norm2() + v.shift(1.0, 2.0) + i = i + 1 + return total diff --git a/scripts/native-bench/programs/prefix.py b/scripts/native-bench/programs/prefix.py new file mode 100644 index 0000000000..860f0626b3 --- /dev/null +++ b/scripts/native-bench/programs/prefix.py @@ -0,0 +1,29 @@ +"""a running sum appended into a list the function owns + +the growth case, and the only benchmark here that iterates a list with `for` +rather than by index — the two are different lowerings and `dot` covers the +other one +""" + + +def prefix(xs: list[float]) -> float: + out = [] + running = 0.0 + for x in xs: + running = running + x + out.append(running) + return out[len(out) - 1] + + +def bench() -> float: + xs = [] + i = 0 + while i < 100000: + xs.append(i * 0.001) + i = i + 1 + total = 0.0 + r = 0 + while r < 5: + total = total + prefix(xs) + r = r + 1 + return total diff --git a/scripts/native-bench/programs/recurse.py b/scripts/native-bench/programs/recurse.py new file mode 100644 index 0000000000..28ac8a8ac2 --- /dev/null +++ b/scripts/native-bench/programs/recurse.py @@ -0,0 +1,16 @@ +"""calls that nest rather than repeat + +`calls` measures a call from a loop, where the caller's frame is reused all the +way down. this one measures depth: every call is live while the next is made, so +it is about the stack the compiler builds rather than about the call sequence +""" + + +def fib(n: int) -> int: + if n < 2: + return n + return fib(n - 1) + fib(n - 2) + + +def bench() -> int: + return fib(24) diff --git a/scripts/native-bench/programs/sets.py b/scripts/native-bench/programs/sets.py new file mode 100644 index 0000000000..7ef4fc08e4 --- /dev/null +++ b/scripts/native-bench/programs/sets.py @@ -0,0 +1,33 @@ +"""a set built once, then tested for membership far more often than it grew + +the hash container without a value, which `dict` does not stand in for: the +membership test is the whole operation rather than a step before a read. the +shape is a short build and a long test phase, so the answer is about `in` +""" + + +def build(n: int) -> set[int]: + seen: set[int] = set() + i = 0 + while i < n: + seen.add(i * 3) + i = i + 1 + return seen + + +def hits(seen: set[int], n: int, passes: int) -> int: + found = 0 + p = 0 + while p < passes: + i = 0 + while i < n: + if i in seen: + found = found + 1 + i = i + 1 + p = p + 1 + return found + + +def bench() -> int: + seen = build(2000) + return hits(seen, 2000, 30) diff --git a/scripts/native-bench/programs/sieve.py b/scripts/native-bench/programs/sieve.py new file mode 100644 index 0000000000..8032ec2145 --- /dev/null +++ b/scripts/native-bench/programs/sieve.py @@ -0,0 +1,30 @@ +"""a sieve: a `bool` buffer the function owns, written from a nested loop + +indexed *writes*, which `dot` and `prefix` between them do not cover — and a +list of `bool`, which is the element type a compiler is most tempted to pack and +most likely to get wrong +""" + + +def sieve(limit: int) -> int: + flags = [] + i = 0 + while i < limit: + flags.append(True) + i = i + 1 + + count = 0 + n = 2 + while n < limit: + if flags[n]: + count = count + 1 + m = n + n + while m < limit: + flags[m] = False + m = m + n + n = n + 1 + return count + + +def bench() -> int: + return sieve(120000) diff --git a/scripts/native-bench/programs/strops.py b/scripts/native-bench/programs/strops.py new file mode 100644 index 0000000000..e360e4df57 --- /dev/null +++ b/scripts/native-bench/programs/strops.py @@ -0,0 +1,29 @@ +"""the string methods real code calls: `split`, `join`, `startswith`, `upper` + +`words` concatenates and `chars` indexes; between them they cover neither of the +two things python code actually does to a string, which is to take it apart and +put it back together through methods on `str` +""" + + +def normalise(line: str) -> int: + parts = line.split(" ") + kept = [] + for part in parts: + if part.startswith("w"): + kept.append(part.upper()) + return len("-".join(kept)) + + +def run(line: str, passes: int) -> int: + total = 0 + p = 0 + while p < passes: + total = total + normalise(line) + p = p + 1 + return total + + +def bench() -> int: + unit = "word0 word1 word2 zero3 word4 word5 zero6 word7 word8 word9" + return run(unit * 200, 60) diff --git a/scripts/native-bench/programs/tuples.py b/scripts/native-bench/programs/tuples.py new file mode 100644 index 0000000000..b1a5245932 --- /dev/null +++ b/scripts/native-bench/programs/tuples.py @@ -0,0 +1,25 @@ +"""tuples built, unpacked and indexed, and a function that returns two values + +the container real code reaches for when it wants a pair, and the only way a +python function returns more than one thing. a tuple is immutable and of known +length, which is a different problem from `list` and is not covered by it +""" + + +def split(value: int) -> tuple[int, int]: + return value // 7, value % 7 + + +def run(n: int) -> int: + total = 0 + i = 0 + while i < n: + whole, part = split(i) + pair = (whole, part) + total = total + pair[0] + pair[1] + i = i + 1 + return total + + +def bench() -> int: + return run(300000) diff --git a/scripts/native-bench/programs/words.py b/scripts/native-bench/programs/words.py new file mode 100644 index 0000000000..4a0420eab6 --- /dev/null +++ b/scripts/native-bench/programs/words.py @@ -0,0 +1,19 @@ +"""string concatenation in a loop, and nothing else + +building and scanning are separate problems with separate answers, so `chars` +scans a string of the same shape and this one only builds. measuring them +together hides whichever is cheaper +""" + + +def build(n: int) -> str: + out = "" + i = 0 + while i < n: + out = out + "word" + str(i % 10) + " " + i = i + 1 + return out + + +def bench() -> int: + return len(build(20000)) diff --git a/scripts/native-bench/timer.py b/scripts/native-bench/timer.py new file mode 100644 index 0000000000..398f247551 --- /dev/null +++ b/scripts/native-bench/timer.py @@ -0,0 +1,165 @@ +"""the timed half of the native benchmark suite, run under the target interpreter + +`bench.py` stages and builds; this imports what it built and times it. the two +are separate programs because everything here has to happen inside one process: +the whole method rests on the four builds of a benchmark being timed in the same +process, in the same wall-clock window, round by round, so that a load spike +lands on all of them rather than on whichever one happened to be running. + +that is possible at all only because each build is staged under a module name of +its own — `mandel_cpython`, `mandel_by`, `mandel_control`, `mandel_mypyc` — since +an extension module's init hook is found by name and two of them cannot answer to +the same one. + +nothing here is timed until it has proved what it is. `load` refuses a module +whose file is not under this run's own root, is older than this run's build, or — +for a build that is supposed to be compiled — does not end in a real extension +suffix. a stale artefact and a build that silently did not happen are the two +ways this suite has lied before, and both are refusals rather than warnings. +""" + +from __future__ import annotations + +import gc +import importlib +import importlib.machinery +import json +import math +import pathlib +import sys +import time + +# the interpreter puts *this script's* directory first, and a benchmark that +# shared a name with anything beside it would be shadowed. nothing is imported +# from here, so the entry only costs correctness +sys.path.pop(0) + + +class Refused(Exception): + """a build that cannot be proved to be the one that was just made""" + + +def load(leg: dict, root: str, built_after: float): + """import one build, or refuse to""" + sys.path.insert(0, leg["dir"]) + try: + module = importlib.import_module(leg["module"]) + except Exception as error: + raise Refused(f"import failed: {type(error).__name__}: {error}") from error + + origin = getattr(module, "__file__", None) + if origin is None: + raise Refused("the module has no __file__, so what ran cannot be identified") + path = pathlib.Path(origin).resolve() + + if not path.is_relative_to(pathlib.Path(root).resolve()): + raise Refused(f"imported {path}, which is outside this run's root") + if path.stat().st_mtime < built_after: + raise Refused(f"imported {path.name}, which predates this run's build") + + suffixes = tuple(importlib.machinery.EXTENSION_SUFFIXES) + if leg["compiled"]: + if not path.name.endswith(suffixes): + raise Refused(f"imported {path.name}, which is not an extension module") + elif path.name.endswith(suffixes): + raise Refused(f"imported {path.name}, which is an extension module") + + if not hasattr(module, "bench"): + raise Refused("the module has no bench()") + return module, str(path) + + +def sample(module, count: int) -> float: + """one timed sample: `count` calls, with the collector quiesced first + + `gc.collect()` sits outside the timed region rather than being disabled, + because disabling it would change what an allocation-heavy benchmark + measures. quiescing it only moves the collection that was going to happen + anyway out of whichever build was unlucky enough to trigger it + """ + gc.collect() + start = time.perf_counter() + for _ in range(count): + module.bench() + return time.perf_counter() - start + + +def main() -> int: + spec = json.loads(pathlib.Path(sys.argv[1]).read_text()) + root, built_after = spec["root"], spec["built_after"] + + loaded, refused, answers, origins = [], {}, {}, {} + for leg in spec["legs"]: + try: + module, origin = load(leg, root, built_after) + except Refused as refusal: + refused[leg["name"]] = str(refusal) + continue + loaded.append((leg["name"], module)) + origins[leg["name"]] = origin + + if spec["mode"] == "probe": + for name, module in loaded: + try: + answers[name] = repr(module.bench()) + except Exception as error: + refused[name] = f"bench() raised {type(error).__name__}: {error}" + print(json.dumps({"answers": answers, "refused": refused, "origins": origins})) + return 0 + + if spec["mode"] == "calibrate": + # how many calls make one sample. every build of a benchmark runs the + # same number, so the pairing stays exact — which means one number has + # to suit builds that can be forty times apart. it is chosen from both + # ends: long enough that the fastest build's sample is well clear of the + # clock, short enough that the slowest build's round does not dominate + # the run. where the two disagree the cap wins, and the control column + # then says out loud whether the resulting sample was too short + per_call = {} + for name, module in loaded: + count = 1 + while True: + elapsed = sample(module, count) + if elapsed >= spec["probe_target"] or count >= spec["max_count"]: + break + grow = max(2, min(8, int(spec["probe_target"] / max(elapsed, 1e-9)))) + count = min(count * grow, spec["max_count"]) + per_call[name] = elapsed / count + + fastest, slowest = min(per_call.values()), max(per_call.values()) + wanted = math.ceil(spec["min_sample"] / fastest) + cap = max(1, int(spec["max_sample"] / slowest)) + count = max(1, min(wanted, cap, spec["max_count"])) + print(json.dumps({"count": count, "per_call": per_call, "refused": refused})) + return 0 + + count, rounds, warmup = spec["count"], spec["rounds"], spec["warmup"] + order = [name for name, _ in loaded] + modules = dict(loaded) + timings = {name: [] for name in order} + + for index in range(warmup + rounds): + # rotate, so no build is systematically the one that pays for a cold + # cache at the top of a round. deterministic rather than random: a + # benchmark run should be reproducible + shift = index % len(order) + for name in order[shift:] + order[:shift]: + elapsed = sample(modules[name], count) + if index >= warmup: + timings[name].append(elapsed / count) + + print( + json.dumps( + { + "timings": timings, + "refused": refused, + "origins": origins, + "count": count, + } + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/native-sweeps/buildsweep.sh b/scripts/native-sweeps/buildsweep.sh index e63175ac77..c9f385ff1b 100755 --- a/scripts/native-sweeps/buildsweep.sh +++ b/scripts/native-sweeps/buildsweep.sh @@ -15,8 +15,8 @@ trap 'rm -rf "$root"' EXIT for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - err=$(cd "$d" && PYTHON="$PY" "$BY" compile m.py -o o 2>&1 >/dev/null) + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + err=$(cd "$d" && PYTHON="$PY" "$BY" compile "$SWEEP_SRC" -o o 2>&1 >/dev/null) if echo "$err" | grep -qiE 'panicked|internal error|stack overflow'; then printf '%s\tPANIC\t%s\n' "$b" "$(echo "$err" | head -1)" >> "$OUT" elif sweep_built "$d"; then diff --git a/scripts/native-sweeps/census.sh b/scripts/native-sweeps/census.sh new file mode 100755 index 0000000000..6a3f3d972f --- /dev/null +++ b/scripts/native-sweeps/census.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# compile every module with --annotate and collect the whole report per module +# usage: census.sh SP BY PY OUT [MODULE...] +SP="$1"; BY="$2"; PY="$3"; OUT="$4"; shift 4 +# shellcheck source=scripts/native-sweeps/sweeplib.sh +. "$(dirname "$0")/sweeplib.sh" +LIB=$(sweep_lib "$PY") +root="$SP/census.$$"; rm -rf "$root"; mkdir -p "$root" +trap 'rm -rf "$root"' EXIT +: > "$OUT" +for b in $(sweep_modules "$LIB" "$@"); do + f="$LIB/$b" + [ -f "$f" ] || continue + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" --annotate --emit-c-only + report="$(sweep_out_dir "$d")/m.annotated" + if [ ! -f "$report" ]; then printf '%s\tno-report\n' "$b" >> "$OUT"; continue; fi + awk -v b="$b" ' + /^[0-9]+ compiled, [0-9]+ left interpreted$/ { print b "\tcounts\t" $1 "\t" $3 } + /^- / && insec==1 { sub(/^- /,""); print b "\tdecline\t" $0 } + /^## left to the interpreted definition/ { insec=1; next } + /^## / && !/^## left to the interpreted definition/ { insec=0 } + /^## class / { print b "\tclass\t" $3 } + ' "$report" >> "$OUT" +done diff --git a/scripts/native-sweeps/instancecensus.sh b/scripts/native-sweeps/instancecensus.sh index 67b0b59de9..fdd3dcf8d8 100755 --- a/scripts/native-sweeps/instancecensus.sh +++ b/scripts/native-sweeps/instancecensus.sh @@ -25,9 +25,20 @@ trap 'rm -rf "$root"' EXIT : > "$OUT" cat > "$root/probe.py" <<'PYEOF' +import importlib +import os import signal import sys +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where an +# interpreted one answers `pkg.m`. this census reads a compiled leg only, and both +# spellings mean "defined by the module under test" +SELF = (MOD, MOD.rpartition('.')[2]) + def _ring(signum, frame): raise TimeoutError('timed out') @@ -35,9 +46,9 @@ def _ring(signum, frame): signal.signal(signal.SIGALRM, _ring) try: - signal.alarm(30) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - import m + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: @@ -51,7 +62,7 @@ counts = [0, 0] def note(value, bucket, slot): kind = type(value) - if getattr(kind, '__module__', None) != 'm': + if getattr(kind, '__module__', None) not in SELF: return claimed = getattr(m, kind.__name__, None) if not isinstance(claimed, type): @@ -99,19 +110,24 @@ PYEOF for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi # the swap this defect is about, read off the emitted C: every class whose name the # module init rebinds to a compiled type - emitted=$(grep -oE 'PyDict_SetItemString\(dict, "[^"]+", By_[A-Za-z0-9_]+_OBJ\)' "$d/o/m.c" \ + # + # read *before* `sweep_place`: a package member's C sits inside the package's place in + # the output tree, and `sweep_place` lays the twin's copy of the package over the top + emitted=$(grep -oE 'PyDict_SetItemString\(dict, "[^"]+", By_[A-Za-z0-9_]+_OBJ\)' \ + "$(sweep_out_dir "$d")/m.c" \ | sed -E 's/.*dict, "([^"]+)".*/\1/' | LC_ALL=C sort -u | paste -sd, -) - cp "$root/probe.py" "$d/o/probe.py" + sweep_place "$d" + cp "$root/probe.py" "$SWEEP_RUN_C/probe.py" # the comma list becomes one argument per class; an array says that, where a bare # command substitution only word-splits by accident names=() [ -n "$emitted" ] && IFS=',' read -r -a names <<< "$emitted" - out=$(cd "$d/o" && "$PY" probe.py "${names[@]}" 2>&1) + out=$(cd "$SWEEP_RUN_C" && "$PY" probe.py "${names[@]}" 2>&1) case "$out" in IMPORT-FAILED*) printf '%s\timport-failed\temitted=%s\n' "$b" "$emitted" >> "$OUT"; continue ;; esac diff --git a/scripts/native-sweeps/isoconstruct.sh b/scripts/native-sweeps/isoconstruct.sh index f3ccd352ea..55a6daaeca 100755 --- a/scripts/native-sweeps/isoconstruct.sh +++ b/scripts/native-sweeps/isoconstruct.sh @@ -23,53 +23,77 @@ trap 'rm -rf "$root"' EXIT cat > "$root/drive.py" <<'PYEOF' # every class the module itself defines, called with no arguments. `signal.alarm` # bounds a constructor that blocks, so one module cannot stall the sweep +import importlib +import os import signal import sys +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings mean "defined by the module under +# test", and nothing else in the staged tree answers to either +SELF = (MOD, MOD.rpartition('.')[2]) + + class _Slow(Exception): pass + def _ring(signum, frame): raise _Slow('timed out') -signal.signal(signal.SIGALRM, _ring) -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. so the failure is caught -# and reported as one line that carries no path at all -try: - signal.alarm(30) - try: - import m - finally: - signal.alarm(0) -except BaseException as error: - print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) - raise SystemExit(0) - -names = [ - name - for name in sorted(vars(m)) - if isinstance(vars(m)[name], type) and getattr(vars(m)[name], '__module__', None) == 'm' -] - -start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 - -for name in names[start:]: - value = vars(m)[name] +def main(): + signal.signal(signal.SIGALRM, _ring) + + # a module can still fail this import — a compiled extension that did not load, a + # C accelerator this build has no source for. both legs fail it alike, but an + # uncaught traceback would not read alike: the interpreted frame names `m.py` where + # the compiled one, running its fallback source through `PyRun_String`, names + # ``. so the failure is caught and reported as one line carrying no path try: - signal.alarm(2) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - made = value() + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: - print(name, type(error).__name__, str(error), flush=True) - else: - print(name, 'built', type(made).__name__, flush=True) + print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) + raise SystemExit(0) + + names = [ + name + for name in sorted(vars(m)) + if isinstance(vars(m)[name], type) + and getattr(vars(m)[name], '__module__', None) in SELF + ] + + start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + + for name in names[start:]: + value = vars(m)[name] + try: + signal.alarm(2) + try: + made = value() + finally: + signal.alarm(0) + except BaseException as error: + print(name, type(error).__name__, str(error), flush=True) + else: + print(name, 'built', type(made).__name__, flush=True) + + +# `multiprocessing` starts a worker by re-running this interpreter and importing this +# file, as `__mp_main__`. a constructor that makes a pool is reached now that the +# package modules import, and without the guard every worker would import the module +# and construct every class in it again — pool included +if __name__ == '__main__': + main() PYEOF # run one leg to completion, restarting past whatever killed it. a constructor that @@ -94,23 +118,31 @@ leg() { for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" # a traceback names the file it came from, and the two legs run from different # directories — so the *path* would read as a difference the module never had - i=$(leg "$d") - c=$(leg "$d/o") + i=$(leg "$SWEEP_RUN_I") + c=$(leg "$SWEEP_RUN_C") if [ "$i" = "$c" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(printf '%s' "$i" | grep -c '')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # a constructor that outran its two seconds on one leg and not the other, or an + # import that outran its thirty, says nothing about the compiler — a loaded machine + # loses either bound. kept out of `differing` so the headline number is the same on + # a busy machine as an idle one; it was not, and two agents reading the same tree + # got different counts + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(printf '%s' "$i") <(printf '%s' "$c") | awk -v b="$b" '{print b "\t| " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isoimport.sh b/scripts/native-sweeps/isoimport.sh index a402eb2690..2b1e1df01b 100755 --- a/scripts/native-sweeps/isoimport.sh +++ b/scripts/native-sweeps/isoimport.sh @@ -15,21 +15,51 @@ root="$SP/isoimp.$$"; rm -rf "$root"; mkdir -p "$root" trap 'rm -rf "$root"' EXIT : > "$OUT" -# an import that never returns would stall the whole sweep, and a subpackage module is -# far likelier to start something than a top-level one. the alarm is left at its default -# disposition on purpose: it kills the leg, both legs are killed alike, and a killed pair -# reads as `same` rather than as a difference nobody can act on -probe='import signal; signal.alarm(30); import m' +# an import that never returns would stall the whole sweep, and a package member is far +# likelier to start something than a top-level module. the alarm is left at its default +# disposition on purpose: it kills the leg, and a leg killed by it says nothing on the +# way out — so what tells the two cases apart is the *exit status*, not the output. +# +# a leg that dies prints nothing, and so does a leg that imports cleanly. reading only +# the text, this rung counted a killed leg as an import that worked — the one reading +# that must never be given, because it is agreement with a leg that answered nothing +# +# the name comes from the staging: `m` for a top-level module, `pkg.m` for a package +# member. both legs are handed the same one +probe='import importlib, os, signal; signal.alarm(int(os.environ["SWEEP_IMPORT_BOUND"])); importlib.import_module(os.environ["SWEEP_MOD"])' + +# one leg, into LEG_STATUS and LEG_TEXT. the status has to come from the interpreter +# rather than from the end of a pipeline, so the text is trimmed afterwards. a leg that +# says nothing is the case this exists for, so the two are returned apart rather than +# packed into one string an empty half would collapse +leg() { + local dir="$1" out + out=$(cd "$dir" && "$PY" -c "$probe" 2>&1); LEG_STATUS=$? + # the two legs run from different directories, so a message that names a file would + # read as a difference the module never had. each is made relative to its own root + LEG_TEXT=$(printf '%s' "$out" | tail -1 | sed "s|$dir/||g") +} for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - i=$(cd "$d" && "$PY" -c "$probe" 2>&1 | tail -1) - c=$(cd "$d/o" && "$PY" -c "$probe" 2>&1 | tail -1) - if [ "$i" = "$c" ]; then printf '%s\tsame\t%s\n' "$b" "$i" >> "$OUT" - else printf '%s\tDIFFERS\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$i" "$c" >> "$OUT"; fi + sweep_place "$d" + leg "$SWEEP_RUN_I"; istat=$LEG_STATUS; i=$LEG_TEXT + leg "$SWEEP_RUN_C"; cstat=$LEG_STATUS; c=$LEG_TEXT + # 142 is SIGALRM: the import outran the bound. a loaded machine loses it on one leg + # and not the other, which says nothing about the compiler — so it is its own + # category rather than a difference or an agreement + if [ "$istat" = 142 ] || [ "$cstat" = 142 ]; then + printf '%s\ttimed-out\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$istat" "$cstat" + elif [ "$istat" -gt 128 ] || [ "$cstat" -gt 128 ]; then + # killed by something else: a leg that died mid-import has not answered, and the + # empty text it leaves behind must not be read as agreement + printf '%s\tDIED\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$istat" "$cstat" + elif [ "$i" = "$c" ]; then printf '%s\tsame\t%s\n' "$b" "$i" + else printf '%s\tDIFFERS\tinterpreted[%s]\tcompiled[%s]\n' "$b" "$i" "$c" + fi >> "$OUT" done -echo "walked: $(wc -l < "$OUT") exercised: $(grep -c $'\tsame\t$' "$OUT") differing: $(grep -c DIFFERS "$OUT") import-failed: $(grep -cE $'\tsame\t.' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(wc -l < "$OUT") exercised: $(grep -c $'\tsame\t$' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") died: $(grep -c $'\tDIED' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -cE $'\tsame\t.' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isosubclass.sh b/scripts/native-sweeps/isosubclass.sh index b14a7b1444..4a432a921a 100755 --- a/scripts/native-sweeps/isosubclass.sh +++ b/scripts/native-sweeps/isosubclass.sh @@ -29,10 +29,22 @@ trap 'rm -rf "$root"' EXIT cat > "$root/drive.py" <<'PYEOF' # every class the module itself defines, derived from. `signal.alarm` bounds a # metaclass that blocks, so one module cannot stall the sweep +import importlib +import os import signal import sys import types +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings mean "defined by the module under +# test", and nothing else in the staged tree answers to either +SELF = (MOD, MOD.rpartition('.')[2]) + class _Slow(Exception): pass @@ -42,51 +54,59 @@ def _ring(signum, frame): raise _Slow('timed out') -signal.signal(signal.SIGALRM, _ring) - -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. and a non-zero exit would -# send the restart loop below round forty times for nothing. so the failure is caught -# and reported as one line that carries no path at all -try: - signal.alarm(30) - try: - import m - finally: - signal.alarm(0) -except BaseException as error: - print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) - raise SystemExit(0) - -start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 -names = [ - name - for name in sorted(vars(m)) - if isinstance(vars(m)[name], type) and getattr(vars(m)[name], '__module__', None) == 'm' -] - - def _body(namespace): # a member the metaclass may want to do something with, which is what reaches # the descriptor protocol at all namespace['probe'] = 1 -for name in names[start:]: - value = vars(m)[name] +def main(): + signal.signal(signal.SIGALRM, _ring) + + # a module can still fail this import — a compiled extension that did not load, a + # C accelerator this build has no source for. both legs fail it alike, but an + # uncaught traceback would not read alike: the interpreted frame names `m.py` where + # the compiled one, running its fallback source through `PyRun_String`, names + # ``. and a non-zero exit would send the restart loop below round forty + # times for nothing. so the failure is caught and reported as one line carrying no + # path at all try: - signal.alarm(2) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - made = types.new_class('Sub', (value,), {}, _body) + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: - print(name, type(error).__name__, str(error), flush=True) - else: - print(name, 'subclassed', made.__name__, flush=True) + print('IMPORT-FAILED', type(error).__name__, str(error), flush=True) + raise SystemExit(0) + + start = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + names = [ + name + for name in sorted(vars(m)) + if isinstance(vars(m)[name], type) + and getattr(vars(m)[name], '__module__', None) in SELF + ] + + for name in names[start:]: + value = vars(m)[name] + try: + signal.alarm(2) + try: + made = types.new_class('Sub', (value,), {}, _body) + finally: + signal.alarm(0) + except BaseException as error: + print(name, type(error).__name__, str(error), flush=True) + else: + print(name, 'subclassed', made.__name__, flush=True) + + +# `multiprocessing` starts a worker by re-running this interpreter and importing this +# file, as `__mp_main__`. the package modules import now, so a module body that starts +# one is reachable, and without the guard the worker would walk the module again +if __name__ == '__main__': + main() PYEOF # run one leg to completion, restarting past whatever killed it @@ -112,21 +132,29 @@ leg() { for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" - i=$(leg "$d") - c=$(leg "$d/o") + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" + i=$(leg "$SWEEP_RUN_I") + c=$(leg "$SWEEP_RUN_C") if [ "$i" = "$c" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(printf '%s' "$i" | grep -c '')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # a class the metaclass took more than two seconds over, or an import that outran + # its thirty — a loaded machine loses either bound on one leg and not the other, + # which says nothing about the compiler. kept out of `differing` so the headline + # number is the same on a busy machine as an idle one, and out of `same` because a + # leg that was killed answered nothing + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(printf '%s' "$i") <(printf '%s' "$c") | awk -v b="$b" '{print b "\t| " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") crashed: $(grep -c 'DIED signal' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/isosurface.sh b/scripts/native-sweeps/isosurface.sh index fb90df22f9..44ee5d233b 100755 --- a/scripts/native-sweeps/isosurface.sh +++ b/scripts/native-sweeps/isosurface.sh @@ -23,6 +23,11 @@ # - every module-level value whose type this module owns is asked whether it is still # an instance of the class now under that name # +# one cause currently dominates `differing`: an emitted class in a package reports its +# `__module__` without the package, so 215 of the 246 differ on nothing else. that is one +# defect, not 215 — `grep '| > .* named ' | grep -v \\.` separates them, and the rest is +# the number to watch +# # usage: isosurface.sh SP BY PY OUT [MODULE...] SP="$1"; BY="$2"; PY="$3"; OUT="$4"; shift 4 # shellcheck source=scripts/native-sweeps/sweeplib.sh @@ -33,8 +38,22 @@ trap 'rm -rf "$root"' EXIT : > "$OUT" cat > "$root/drive.py" <<'PYEOF' +import importlib +import os import signal +# the staging says what to import: `m` for a top-level module, `pkg.m` for a package +# member, which is the only name its relative imports resolve against. both legs are +# handed the same one +MOD = os.environ['SWEEP_MOD'] +# `by compile` names a module after its file, and an emitted class takes its +# `__module__` from the last component of that name — so it answers `m` where its +# interpreted twin answers `pkg.m`. both spellings have to select the *same classes*, or +# the compiled leg would appear to define none of them. what a class says its module is +# stays compared outright, because that answer is wrong rather than merely differently +# spelled +SELF = (MOD, MOD.rpartition('.')[2]) + class _Slow(Exception): pass @@ -46,16 +65,15 @@ def _ring(signum, frame): signal.signal(signal.SIGALRM, _ring) -# a subpackage module is far likelier than a top-level one to fail this import: a -# `from . import x` has no parent package to resolve against once the file has been -# copied out on its own. both legs fail it alike, but an uncaught traceback would not -# read alike — the interpreted frame names `m.py` where the compiled one, running its -# fallback source through `PyRun_String`, names ``. so the failure is caught -# and reported as one line that carries no path at all +# a module can still fail this import — a compiled extension that did not load, a C +# accelerator this build has no source for. both legs fail it alike, but an uncaught +# traceback would not read alike: the interpreted frame names `m.py` where the compiled +# one, running its fallback source through `PyRun_String`, names ``. so the +# failure is caught and reported as one line that carries no path at all try: - signal.alarm(30) + signal.alarm(int(os.environ['SWEEP_IMPORT_BOUND'])) try: - import m + m = importlib.import_module(MOD) finally: signal.alarm(0) except BaseException as error: @@ -82,7 +100,7 @@ def value_of(target, key): def owned(value): - return isinstance(value, type) and getattr(value, '__module__', None) == 'm' + return isinstance(value, type) and getattr(value, '__module__', None) in SELF # what a compiled type does not carry, by construction rather than by defect: a spec @@ -101,6 +119,10 @@ for name, cls in classes: try: show('%s mro %s' % (name, [base.__name__ for base in cls.__mro__])) show('%s meta %s' % (name, type(cls).__name__)) + # `__module__` is compared outright, and the two spellings above are *not* + # collapsed here. a compiled class in a package answers `m` where its twin + # answers `pkg.m`, and that is a wrong answer rather than a house style: + # `dataclasses` does `sys.modules[cls.__module__].__dict__` and gets `None` show('%s named %s %s' % (name, cls.__module__, cls.__qualname__)) # one way: a name the interpreted class has and the compiled one does not. # `MISSING` is printed by the compiled leg only, so an identical pair of legs @@ -127,7 +149,7 @@ for name in sorted(vars(m)): if isinstance(value, type): continue kind = type(value) - if getattr(kind, '__module__', None) != 'm': + if getattr(kind, '__module__', None) not in SELF: continue claimed = getattr(m, kind.__name__, None) if not isinstance(claimed, type): @@ -142,29 +164,35 @@ PYEOF for b in $(sweep_modules "$LIB" "$@"); do f="$LIB/$b" [ -f "$f" ] || continue - d="$root/w"; sweep_stage "$d" "$f" - sweep_compile "$d" "$PY" "$BY" + d="$root/w"; sweep_stage "$d" "$LIB" "$b" + sweep_compile "$b" "$d" "$PY" "$BY" if ! sweep_built "$d"; then printf '%s\tno-artifact\n' "$b" >> "$OUT"; continue; fi - cp "$root/drive.py" "$d/drive.py"; cp "$root/drive.py" "$d/o/drive.py" + sweep_place "$d" + cp "$root/drive.py" "$SWEEP_RUN_I/drive.py"; cp "$root/drive.py" "$SWEEP_RUN_C/drive.py" # a traceback names the file it came from, and the two legs run from different # directories — so the *path* would read as a difference the module never had - i=$(cd "$d" && "$PY" drive.py 2>&1 | sed "s|$d/||g") - c=$(cd "$d/o" && "$PY" drive.py 2>&1 | sed "s|$d/o/||g") + i=$(cd "$SWEEP_RUN_I" && "$PY" drive.py 2>&1 | sed "s|$SWEEP_RUN_I/||g") + c=$(cd "$SWEEP_RUN_C" && "$PY" drive.py 2>&1 | sed "s|$SWEEP_RUN_C/||g") # a `has` line is one-directional: only its loss counts, so the compiled leg's extra # names are dropped before the comparison and the interpreted leg's are kept ionly=$(echo "$i" | grep -v ' has ') only=$(echo "$c" | grep -v ' has ') lost=$(comm -23 <(echo "$i" | grep ' has ' | sort) <(echo "$c" | grep ' has ' | sort)) if [ "$ionly" = "$only" ] && [ -z "$lost" ]; then - # a module that cannot be imported standalone agrees on both legs and exercises - # nothing — kept apart from `same` so the denominator stays honest + # a module that cannot be imported here agrees on both legs and exercises nothing — + # kept apart from `same` so the denominator stays honest case "$i" in IMPORT-FAILED*) printf '%s\timport-failed\t%s\n' "$b" "$i" ;; *) printf '%s\tsame\t%s\n' "$b" "$(echo "$i" | wc -l | tr -d ' ')" ;; esac >> "$OUT" + elif printf '%s%s' "$i" "$c" | grep -q '_Slow timed out'; then + # the import bound is 30 seconds and a loaded machine loses it on one leg and not + # the other. that says nothing about the compiler, so it is kept out of `differing` + # — and out of `same`, because a leg that was killed answered nothing + printf '%s\ttimed-out\n' "$b" >> "$OUT" else printf '%s\tDIFFERS\n' "$b" diff <(echo "$ionly") <(echo "$only") | awk -v b="$b" '{print b "\t| " $0}' echo "$lost" | awk -v b="$b" 'NF {print b "\t| lost " $0}' fi >> "$OUT" done -echo "walked: $(grep -cE $'\t(same|DIFFERS|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") lost: $(grep -c $'\t| lost ' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" +echo "walked: $(grep -cE $'\t(same|DIFFERS|timed-out|import-failed|no-artifact)' "$OUT") exercised: $(grep -cE $'\t(same|DIFFERS)' "$OUT") differing: $(grep -c $'\tDIFFERS' "$OUT") lost: $(grep -c $'\t| lost ' "$OUT") timed-out: $(grep -c $'\ttimed-out' "$OUT") import-failed: $(grep -c $'\timport-failed' "$OUT") no-artifact: $(grep -c $'\tno-artifact' "$OUT")" diff --git a/scripts/native-sweeps/sweeplib.sh b/scripts/native-sweeps/sweeplib.sh index 14532f4441..7451368ec8 100755 --- a/scripts/native-sweeps/sweeplib.sh +++ b/scripts/native-sweeps/sweeplib.sh @@ -43,45 +43,246 @@ sweep_modules() { | LC_ALL=C sort } +# the modules whose type inference does not terminate today +# +# they are still walked — a rung that skipped them would stop noticing the day they +# are fixed — but on a short leash, because the full bound is 180s and re-proving a +# known hang once per rung costs a quarter hour of every sweep cycle +# +# it is empty. three were on this list on the strength of "produced no artefact", +# and the stale-entry alarm caught all three on its first real run: on +# `origin/main` as much as here, `bdb.py` overflows the stack in 5s, and +# `pickletools.py` and `profile.py` finish in 5s with a salsa panic reported as a +# diagnostic. none of those is a hang, and none of them needs a leash +# +# `ast.py` came off when the return type it grew a constructor a round was bounded; +# it now compiles in 2s. `turtle.py` came off when a tuple whose elements were the +# cycle's own marker stopped alternating between the widened and the marked form of +# itself; it now compiles in 24s +# +# `${VAR-default}` rather than `${VAR:-default}`: an explicitly empty list is a +# caller saying "treat every hang as new", which is how this gets tested +BY_KNOWN_HANGS=${BY_KNOWN_HANGS-""} + # compile one staged module, bounded # -# the compiler is unbounded and a module whose type inference does not terminate wedges -# a whole rung: `ast.py` alone held one run for 21 minutes with no progress. the five -# that do this today (`ast`, `bdb`, `pickletools`, `profile`, `turtle`) stopped -# terminating when the branch was rebased, and the cause is on main rather than here — -# so the rungs have to survive it rather than wait for it. a bounded module reports as -# `no-artifact`, which is what it is: the sweep learned nothing about it +# a module that has never terminated needs only long enough to confirm it still does +# not; anything else gets the full bound. the two cases that must never pass quietly: +# a module *not* on the list that hits its bound is a new hang, and a module *on* the +# list that finishes means the list is stale. both say so on stderr and land in +# `$OUT.alarms`, because a rung that swallows either stops being evidence # -# SWEEP_BOUND is seconds; 0 disables the bound +# SWEEP_BOUND is seconds (0 disables); SWEEP_BOUND_KNOWN is the short leash sweep_compile() { - local dir="$1" py="$2" by="$3" - local bound=${SWEEP_BOUND:-180} + local name="$1" dir="$2" py="$3" by="$4"; shift 4 + local bound=${SWEEP_BOUND:-180} known=0 + case " $BY_KNOWN_HANGS " in *" $name "*) known=1; bound=${SWEEP_BOUND_KNOWN:-15} ;; esac + if [ "$bound" -eq 0 ]; then - (cd "$dir" && PYTHON="$py" "$by" compile m.py -o o >/dev/null 2>&1) + (cd "$dir" && PYTHON="$py" "$by" compile "$SWEEP_SRC" -o o "$@" >/dev/null 2>&1) return $? fi - (cd "$dir" && PYTHON="$py" "$by" compile m.py -o o >/dev/null 2>&1) & + + (cd "$dir" && PYTHON="$py" "$by" compile "$SWEEP_SRC" -o o "$@" >/dev/null 2>&1) & local pid=$! - ( sleep "$bound"; kill -9 "$pid" 2>/dev/null ) & + # the watchdog gets its own descriptors, and its `sleep` is killed with it. + # killing only the subshell leaves the `sleep` orphaned holding the inherited + # stdout, so a rung that finished in a second did not close its pipe until the + # bound elapsed — a reader saw the summary 180s late for no reason at all + { sleep "$bound"; kill -9 "$pid" 2>/dev/null; } >/dev/null 2>&1 & local killer=$! wait "$pid" 2>/dev/null local rc=$? + pkill -9 -P "$killer" 2>/dev/null kill -9 "$killer" 2>/dev/null wait "$killer" 2>/dev/null + + # 137 is the kill; anything else is the compiler's own answer + if [ "$rc" -eq 137 ]; then + if [ "$known" -eq 0 ]; then + sweep_alarm "$name" "did not finish in ${bound}s — a hang that is not on BY_KNOWN_HANGS" + fi + elif [ "$known" -eq 1 ]; then + sweep_alarm "$name" "finished in under ${bound}s — it is on BY_KNOWN_HANGS and should come off" + fi return $rc } -# lay one module out as a project of its own. `m.py` regardless of where it came from, -# so an import of `m` names the same thing in both legs +# an alarm is not a result: it says the sweep itself learned something it cannot +# record in a row, so it goes where it will be seen rather than into the tsv +sweep_alarm() { + printf '!! sweep alarm: %s %s\n' "$1" "$2" >&2 + [ -n "${OUT:-}" ] && printf '%s\t%s\n' "$1" "$2" >> "$OUT.alarms" + return 0 +} + +# the package every staged tree sits inside +# +# a package member has to be staged *in its package* or its relative imports have +# nothing to resolve against. a copy laid out under the package's own name would be +# imported in place of the interpreter's own, and that goes wrong two ways: `encodings` +# is already in `sys.modules` before a probe starts, so `import encodings.m` searches +# the real stdlib and finds nothing; and a copy of `re` or `importlib` first on the path +# answers every later import in the process, the driver's as much as the module's. one +# outer package nobody else names keeps the copy reachable and the interpreter's own +# stdlib intact. it costs the module only a prefix on its `__name__` +SWEEP_WRAP=by_stage + +# nothing the sweep runs benefits from a cache, and a `.pyc` is one more file that +# could answer in place of the one the sweep staged +export PYTHONDONTWRITEBYTECODE=1 + +# how long a leg is given to import the module, in seconds +# +# it is there to stop a module body that never returns, and thirty seconds is far more +# than any of them needs — but not more than the *machine* can take. macos scans a +# freshly written `.so` the first time it is loaded, and every module in the sweep +# builds a new one: on an idle machine that is under a second, and on a loaded one it +# has been measured at twenty-five, with the process asleep for all of it. the same +# extension then loads in 0.04s. so on a busy machine raise this rather than reading the +# timeouts as results +export SWEEP_IMPORT_BOUND=${SWEEP_IMPORT_BOUND:-30} + +# true when every directory above the module is a package +# +# a directory holding python files is not necessarily one: `config-3.13-darwin` has no +# `__init__.py`, and its name is not even an identifier. a module under one of those is +# reached by path rather than by import and is staged on its own, which is what it is +sweep_in_package() { + local lib="$1" rel="$2" prefix="" + case "$rel" in */*) ;; *) return 1 ;; esac + local rest="${rel%/*}" + while [ -n "$rest" ]; do + prefix="$prefix${rest%%/*}" + [ -f "$lib/$prefix/__init__.py" ] || return 1 + case "$rest" in + */*) rest="${rest#*/}"; prefix="$prefix/" ;; + *) rest="" ;; + esac + done + return 0 +} + +# lay one module out as a project of its own, and say where the two legs run +# +# the module is `m.py` wherever it came from, so both legs import the same name and +# `m.c`, `m.annotated` and `m*.so` keep theirs. +# +# a top-level module is staged alone, as it always was. a package member is staged in a +# copy of its whole package, so `from . import x` resolves — for the compiler as much as +# for the two legs. the siblings are on disk but kept out of the project's *file set*: +# `by compile` compiles every source in the project it is run from, and building all 122 +# modules of `encodings` once per member of `encodings` is fifteen thousand builds +# +# the whole package rather than the part the module imports: which part that is, is the +# same import graph the sweep exists to test, and a stage that had to be right about it +# would fail in the direction that hides defects +# +# sets, for the caller: +# SWEEP_MOD the dotted module name both legs import +# SWEEP_SRC the staged source, relative to $dir +# SWEEP_RUN_I the directory the interpreted leg runs from +# SWEEP_RUN_C the directory the compiled leg runs from sweep_stage() { - local dir="$1" src="$2" - rm -rf "$dir"; mkdir -p "$dir"; cp "$src" "$dir/m.py" - printf '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n' > "$dir/pyproject.toml" + local dir="$1" lib="$2" rel="$3" limit="" + rm -rf "$dir" "$dir/o"; mkdir -p "$dir" + export SWEEP_RUN_I="$dir" SWEEP_RUN_C="$dir/o" + if ! sweep_in_package "$lib" "$rel"; then + export SWEEP_MOD=m SWEEP_SRC=m.py + cp "$lib/$rel" "$dir/m.py" + else + local pkg="${rel%%/*}" sub="${rel%/*}" dotted + dotted=$(printf '%s' "$sub" | tr / .) + export SWEEP_SRC="$SWEEP_WRAP/$sub/m.py" + export SWEEP_MOD="$SWEEP_WRAP.$dotted.m" + mkdir -p "$dir/$SWEEP_WRAP" + : > "$dir/$SWEEP_WRAP/__init__.py" + cp -R "$lib/$pkg" "$dir/$SWEEP_WRAP/$pkg" + find "$dir/$SWEEP_WRAP" -name __pycache__ -type d -exec rm -rf {} + 2>/dev/null + cp "$lib/$rel" "$dir/$SWEEP_SRC" + # a few package names are in ty's default `src.exclude` — `venv` above all — so a + # member of one is dropped before anything can compile it, and the rung reports + # `no-artifact` for a module that is perfectly fine. the negation re-includes the + # directory; it is written per-stage rather than as a blanket rule so the exclusion + # still holds for everything the sweep did not deliberately stage + limit=$(printf '[tool.ty.src]\ninclude=["%s"]\nexclude=["!**/%s/"]\n' \ + "$SWEEP_SRC" "${rel%%/*}") + fi + printf '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n%s' "$limit" \ + > "$dir/pyproject.toml" +} + +# where the build wrote the staged module's artefacts +# +# `by compile` lays its output out as the *module* tree, so a package member's `m.c`, +# `m.annotated` and `m*.so` sit at the member's own place under `o` rather than at the +# top of it — which is where the staged source sits within the project +sweep_out_dir() { + local rel="${SWEEP_SRC%/*}" + if [ "$rel" = "$SWEEP_SRC" ]; then printf '%s/o' "$1"; else printf '%s/o/%s' "$1" "$rel"; fi +} + +# put the twin's sources around the extension the compiled leg will import +# +# a package member's compiled leg needs the tree its twin has, and the build has already +# left the extension in the module's place inside it — so the extension is set aside +# while the twin's copy of the package is laid down over the top, then put back. the +# staged `m.py` is *removed* from it: python prefers an extension to a source of the same +# name, so leaving it there would mean an extension that failed to load was replaced by +# the interpreted module — and the leg would then agree with its twin for the one reason +# that makes the whole comparison meaningless +# +# call it after `sweep_built`, before running either leg +# the extension the build left, or nothing when it left none +# +# a glob rather than `ls`: what comes back is a path this file goes on to `mv` and to hand +# to python, and `ls` would mangle any name that needed quoting. an unmatched glob stays +# literal in bash, so the `-e` is what distinguishes "no artefact" from "one named `m*.so`" +sweep_artifact() { + local matches + matches=("$(sweep_out_dir "$1")"/m*.so) + [ -e "${matches[0]}" ] || return 1 + printf '%s\n' "${matches[0]}" +} + +sweep_place() { + local dir="$1" so name + so=$(sweep_artifact "$dir") || so="" + if [ "$SWEEP_MOD" != m ]; then + name=${so##*/} + mv "$so" "$dir/$name" + rm -rf "${dir:?}/o/$SWEEP_WRAP" + cp -R "$dir/$SWEEP_WRAP" "$dir/o/$SWEEP_WRAP" + rm -f "$dir/o/$SWEEP_SRC" + so="$dir/o/${SWEEP_SRC%/*}/$name" + mv "$dir/$name" "$so" + fi + sweep_warm "$so" +} + +# pay the operating system's first-load cost before anything is timed +# +# macos validates a freshly built dylib the first time it is loaded, and the process is +# asleep for all of it: 0.42s on an idle machine against 0.04s for every load after, and +# measured at 17.8s under contention. every module in a sweep builds a new one, so that +# cost lands inside whatever bound the rung set and comes back as a `timed-out` — which +# reads as a result and is not one. a full `isoimport` reported 26 of them; re-running +# exactly those 26 with a larger bound gave 26 exercised and 0 differing, so it was the +# operating system every time +# +# `ctypes.CDLL` loads the object without calling `PyInit_`, so this pays the validation +# without running the module body. that matters: a warm-up that imported the module would +# run its import side effects an extra time, and one that hung would hang here instead +sweep_warm() { + local so="$1" + [ -n "$so" ] && [ -f "$so" ] || return 0 + "$PY" -c 'import ctypes, sys; ctypes.CDLL(sys.argv[1])' "$so" >/dev/null 2>&1 + return 0 } # true when the build actually left an extension module behind. `-d o` is not enough: # a build that fails halfway leaves the directory and no artefact, and the compiled leg # then fails to import for a reason that is not the defect the sweep is looking for sweep_built() { - ls "$1"/o/m*.so >/dev/null 2>&1 + sweep_artifact "$1" >/dev/null } diff --git a/scripts/shrink.py b/scripts/shrink.py new file mode 100755 index 0000000000..e295afa398 --- /dev/null +++ b/scripts/shrink.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""shrink a python file to the smallest one that still provokes a `by` failure + +every cycle panic fixed in this repo was cracked by minimising first: 2714 lines to 21, +5055 to 4, 2904 to 3, 621 to 4. in each case the mechanism guessed beforehand turned out +wrong, and the small reproducer is what settled it. + +usage: + scripts/shrink.py --by [--python ] + [--marker "too many cycle iterations"] [--command check] + +three traps guarded here, each of which produced a confident wrong answer before it was: + +1. every candidate gets its own directory. `by check`/`by compile` operate on the whole + *project*, so an original sitting beside a candidate is analysed together with it — + the predicate is then true no matter what the candidate says, and this "shrank" 2714 + lines to 4 that reproduce nothing at all. + +2. a candidate whose rendered source equals the current best is skipped. after a + successful removal the cached statement holders can still hold nodes now detached from + the tree; mutating those re-renders identical source, and the predicate "passes" + without anything having changed. one run reported 8982 hits against 16 misses with the + line count frozen. + +3. the predicate is checked against a known-failing *and* a known-clean input before any + shrinking starts. `timeout` does not exist on macos, so a script relying on it once + answered "OK" for everything. + +the tell for all three is the same: **a predicate that never says no is not minimising, +it is deleting**. the hit/miss ratio is printed for that reason — a healthy run is on the +order of 15 hits to 40 misses, not 8982 to 16. +""" + +from __future__ import annotations + +import argparse +import ast +import os +import shutil +import subprocess +import sys +import tempfile + +PYPROJECT = '[project]\nname="s"\nversion="0"\nrequires-python=">=3.13"\n' +BODY_FIELDS = ("body", "orelse", "finalbody") + + +class Predicate: + """does `by` still fail the way we are shrinking towards?""" + + def __init__(self, by, python, marker, command, suffix): + self.by, self.python = by, python + self.marker, self.command, self.suffix = marker, command, suffix + self.hits = self.misses = 0 + + def __call__(self, source): + # trap 1: its own directory, holding only this candidate + work = tempfile.mkdtemp(prefix="shrink-") + try: + name = f"m{self.suffix}" + with open(os.path.join(work, name), "w") as handle: + handle.write(source) + with open(os.path.join(work, "pyproject.toml"), "w") as handle: + handle.write(PYPROJECT) + env = {**os.environ, "PYTHON": self.python} + try: + done = subprocess.run( + [self.by, self.command, name], + cwd=work, + env=env, + capture_output=True, + text=True, + timeout=300, + ) + except subprocess.TimeoutExpired: + self.misses += 1 + return False + hit = self.marker in (done.stdout + done.stderr) + self.hits += hit + self.misses += not hit + return hit + finally: + shutil.rmtree(work, ignore_errors=True) + + +def holders(tree): + """every statement list in the tree, as (list, ) — one entry per body""" + found = [] + for node in ast.walk(tree): + for field in BODY_FIELDS: + body = getattr(node, field, None) + if ( + isinstance(body, list) + and body + and all(isinstance(s, ast.stmt) for s in body) + ): + found.append(body) + for handler in getattr(node, "handlers", None) or []: + found.append(handler.body) + return found + + +def shrink(source, still_fails): + """greedily drop statements at any depth while the predicate holds""" + best = source + changed = True + rounds = 0 + while changed: + changed, rounds = False, rounds + 1 + for index in range(len(holders(ast.parse(best)))): + at = 0 + while True: + tree = ast.parse(best) + bodies = holders(tree) + if index >= len(bodies) or at >= len(bodies[index]): + break + body = bodies[index] + body.pop(at) + if not body: + body.append(ast.Pass()) + try: + candidate = ast.unparse(tree) + except (ValueError, AttributeError, RecursionError): + at += 1 + continue + # trap 2: an identical rendering proves nothing, so it is not a trial + if candidate == best: + at += 1 + continue + if still_fails(candidate): + best = candidate # the list shifted down, so `at` stays put + changed = True + else: + at += 1 + print(f"round {rounds}: {len(best.splitlines())} lines", flush=True) + return best + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source") + parser.add_argument("output") + parser.add_argument("--by", required=True, help="a release-built `by`") + parser.add_argument("--python", default=sys.executable) + parser.add_argument("--marker", default="too many cycle iterations") + parser.add_argument( + "--command", default="check", help="check | compile | transpile" + ) + args = parser.parse_args() + + with open(args.source) as handle: + source = handle.read() + suffix = os.path.splitext(args.source)[1] or ".py" + still_fails = Predicate(args.by, args.python, args.marker, args.command, suffix) + + # trap 3: a predicate that cannot say no is not a predicate + if not still_fails(source): + sys.exit(f"the unmodified source does not produce {args.marker!r}") + if still_fails("x = 1\n"): + sys.exit( + "a trivial clean file also matches — the predicate is not discriminating" + ) + print("predicate validated on a failing and a clean input", flush=True) + + # normalise through `unparse` first, so the shrinker's own rendering is the baseline + # and the first successful removal is not credited to a formatting change + base = ast.unparse(ast.parse(source)) + if not still_fails(base): + sys.exit( + "the source stops failing once reformatted — shrink the original by hand" + ) + + best = shrink(base, still_fails) + with open(args.output, "w") as handle: + handle.write(best + "\n") + print( + f"final: {len(best.splitlines())} lines " + f"({still_fails.hits} hits / {still_fails.misses} misses)", + flush=True, + ) + + +if __name__ == "__main__": + main() From dfd914d53e66555ffb3be9862abae01f155db1b6 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:35:56 +1000 Subject: [PATCH 6/7] a benchmark suite, an interpreter matrix in ci, and the docs for both MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the backend had no repeatable way to say whether a change made anything faster, and ci tested it against whatever `python3` the runner image happened to ship. the suite times cpython, this backend and mypyc on the same programs and reports ratios rather than absolutes, because cpython's own numbers move several-fold with machine load. ci now pins the interpreter and covers both supported versions across the platform axis — windows on the newest, because its toolchain path is the most version-sensitive code here (it has to name the interpreter's import library, whose stem carries both the version and the free-threaded `t`), and macos on the floor version. the two genuinely differ: 3.14 works a class's annotations out lazily, so a twin's dict holds `__annotate_func__` rather than a mapping, and it rewrote the wording of several runtime errors. a fix that greens one and breaks the other is not a fix. --- .github/workflows/ci.yaml | 158 ++++++++++++-- Cargo.lock | 1 + docs/basedpython/cli-reference.md | 6 + .../development/compilation/benchmarks.md | 203 ++++++++++++++++++ .../development/compilation/index.md | 2 + .../development/compilation/runtime.md | 32 +++ pyproject.toml | 5 + scripts/native-bench/bench.py | 58 +++-- scripts/native-bench/programs/alloc.py | 2 +- scripts/native-bench/programs/fields.py | 2 +- scripts/native-bench/programs/inherit.py | 2 +- scripts/native-bench/programs/methods.py | 2 +- scripts/native-bench/programs/objects.py | 2 +- zensical.toml | 1 + 14 files changed, 439 insertions(+), 37 deletions(-) create mode 100644 docs/basedpython/development/compilation/benchmarks.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 070026938e..77553c64fa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -305,16 +305,50 @@ jobs: # transitive dep and fails on the api skew. the fork publishes only the # `basedpython` wheel to pypi, never these crates to crates.io - # the project supports 3.13 as well as `PYTHON_VERSION`, and the two differ in what a - # class does with its annotations and in the wording of several exceptions. a job of - # its own rather than a step on `cargo test (linux)`: as a step it pushed that job past - # its budget and the doctest and dogfood steps after it never ran at all - cargo-test-backend-other-python: - name: "cargo test (native backend, python 3.13)" + # `by compile` builds against the running interpreter's headers, links its abi and has + # to answer exactly as that interpreter does, so *which* interpreter is a first-class + # variable and belongs in the workflow rather than in whatever a runner image ships. + # + # the supported set is python 3.13 and 3.14, each in its gil and its free-threaded + # build. `PYTHON_VERSION` is the primary and `cargo test (linux)` runs the whole suite + # against it; this job covers the other three. 3.12 is deliberately absent — it is + # below the project's floor, and it was only ever reached because an image shipped it. + # + # only the version-sensitive targets run here, because only they start an interpreter: + # * the six backend crates, which emit and then load real extension modules + # * `by_transforms`' `*_runtime` tests, which transpile and then execute + # * `ty`'s divergence and end-to-end tests, which do the same + # the rest of the suite reads a vendored typeshed and a configured `python-version`, + # so its answers do not move with the interpreter and repeating it buys nothing. + # + # a job of its own rather than steps on `cargo test (linux)`: as a step it pushed that + # job past its budget and the doctest and dogfood steps after it never ran at all + cargo-test-interpreter: + name: "cargo test (python ${{ matrix.python }})" runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} needs: determine_changes if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} - timeout-minutes: 30 + timeout-minutes: 45 + strategy: + # one interpreter failing says nothing about the others, and which ones are green + # is the whole answer this job exists to give + fail-fast: false + matrix: + include: + - python: "3.13" + backend_filter: "all()" + # the free-threaded builds run the same tests, minus one that cannot be + # observed there: `a_closure_does_not_leak_its_environment` asserts that + # `sys.getrefcount` *rises* while a closure holds a string, and a + # free-threaded interpreter reports the immortal sentinel for that string + # (4294967295 on 3.13t, 3221225472 on 3.14t) whoever holds it. an ordinary + # cpython closure fails the same assertion, so it measures the instrument + # rather than the backend. the leak half of that test — the count is + # unchanged after 20000 closures — runs and passes on both + - python: "3.13t" + backend_filter: "not test(=a_closure_does_not_leak_its_environment)" + - python: "3.14t" + backend_filter: "not test(=a_closure_does_not_leak_its_environment)" env: CARGO_PROFILE_DEV_DEBUG: line-tables-only steps: @@ -338,14 +372,62 @@ jobs: with: version: "0.11.31" enable-cache: "true" + # `PYTHON` is the load-bearing pin: every harness that starts an interpreter reads + # it first. `PATH` is set as well because each of those harnesses falls back to a + # bare name if the variable is ever unset, and `by_transforms` tries `python3.13` + # *before* `python3` — so an unpinned run does not merely drift, it drifts towards + # one particular version. uv's managed installs put a `python3` next to the + # versioned name, and in a free-threaded install that `python3` is itself the + # free-threaded build - name: "Pin the interpreter" + env: + PY: ${{ matrix.python }} + run: | + uv python install "$PY" + python_path="$(uv python find "$PY")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + dirname "$python_path" >> "$GITHUB_PATH" + # a leg that silently resolved to a different interpreter would report a green that + # belongs to some other version, which is the failure mode this whole job is about + - name: "Verify the interpreter is the one asked for" + env: + PY: ${{ matrix.python }} run: | - uv python install 3.13 - echo "PYTHON=$(uv python find 3.13)" >> "$GITHUB_ENV" + echo "PYTHON=$PYTHON" + "$PYTHON" -VV + echo "python3 on PATH: $(command -v python3)" + # a free-threaded leg has to be genuinely free-threaded rather than a gil + # build answering to a `t` name, so the probe reports both facts at once + want_version="${PY%t}" + want="$want_version false" + if [ "$PY" != "$want_version" ]; then + want="$want_version true" + fi + probe='import sys, sysconfig; print("%d.%d" % sys.version_info[:2], "true" if sysconfig.get_config_var("Py_GIL_DISABLED") else "false")' + got=$("$PYTHON" -c "$probe") + echo "PYTHON reports: $got (want: $want)" + [ "$got" = "$want" ] || { echo "::error::PYTHON is not $PY"; exit 1; } + # the harnesses fall back to a bare `python3` when the variable is unset, so + # that has to be the same interpreter too + got_path=$(python3 -c "$probe") + echo "python3 reports: $got_path (want: $want)" + [ "$got_path" = "$want" ] || { echo "::error::python3 on PATH is not $PY"; exit 1; } - name: "Run the native backend" + env: + BACKEND_FILTER: ${{ matrix.backend_filter }} run: | cargo nextest run \ - -p by_build -p by_ir -p by_irbuild -p by_opt -p by_codegen_c -p by_rt + -p by_build -p by_ir -p by_irbuild -p by_opt -p by_codegen_c -p by_rt \ + -E "$BACKEND_FILTER" + # a separate invocation on purpose: the backend's differential tests fork processes + # and compile C, and sharing a run with `by_transforms`' ~1800 other tests starves + # them into failures that pass on their own + - name: "Run the transpiler's runtime tests" + run: cargo nextest run -p by_transforms -E 'binary(/_runtime$/)' + - name: "Run ty's divergence and end-to-end tests" + run: | + cargo nextest run -p ty \ + -E 'binary(mdtest_divergence) + binary(by_e2e) + binary(django_lookup_runtime)' cargo-test-linux: name: "cargo test (linux)" @@ -404,17 +486,24 @@ jobs: # Ignore errors if this step fails; we want to continue to later steps in the workflow anyway. # This step is just to get nice GitHub annotations on the PR diff in the files-changed tab. run: cargo test -p ty_python_semantic --test mdtest || true - # the native backend's tests run against whatever `python3` is first on `PATH`, + # the tests that execute python run against whatever `python3` is first on `PATH`, # which without this is whichever interpreter the runner image happens to ship — # a 3.14 on macos and a 3.12 on linux, so the version under test moved with the # image rather than with the project. `by compile` builds against the running # interpreter's headers and has to answer as that interpreter does, so which one - # it is belongs in the workflow. the other platforms are deliberately left on the - # image's own interpreter: that is coverage on top of this, not instead of it - - name: "Pin the interpreter the native backend is tested against" + # it is belongs in the workflow. + # + # `PATH` as well as `PYTHON`: the harnesses all read the variable first, but each + # falls back to a bare name when it is unset, and `by_transforms` tries + # `python3.13` *before* `python3` — so a run that lost the variable would not + # drift randomly, it would drift to one particular version. the rest of the + # supported set is covered by `cargo test (python …)` + - name: "Pin the interpreter the python-executing tests use" run: | uv python install "${PYTHON_VERSION}" - echo "PYTHON=$(uv python find "${PYTHON_VERSION}")" >> "$GITHUB_ENV" + python_path="$(uv python find "${PYTHON_VERSION}")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + dirname "$python_path" >> "$GITHUB_PATH" - name: "Run tests" run: cargo insta test --all-features --unreferenced reject --test-runner nextest --disable-nextest-doctest - name: "Run doctests" @@ -479,13 +568,30 @@ jobs: - name: "Run doctests" run: cargo test --doc --profile profiling --all-features + # these two used to take whatever interpreter the runner image shipped. that drift did + # find real bugs — a 3.14 macos image surfaced six failures nobody was looking for — + # but it found them as a red that named no version, and a green said nothing about + # which version was covered. the same job could change answer between two reruns of + # the same commit. so the interpreter is pinned here too, and the coverage the drift + # used to buy by accident is bought on purpose by `cargo test (python …)` instead. + # + # the two platforms take different versions, so the platform axis and the version axis + # are both covered without a third job: windows takes the newest because its toolchain + # path is the most version-sensitive code in the backend (it has to name the + # interpreter's import library, whose stem carries both the version and the + # free-threaded `t`), and macos takes the floor version cargo-test-other: strategy: + # each leg is now a distinct platform *and* version, so cancelling the other one + # on the first failure throws away half the answer + fail-fast: false matrix: - platform: - - ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-windows-2022-x86-64-16x32' || 'windows-latest' }} - - ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-macos-15' || 'macos-latest' }} - name: "cargo test (${{ matrix.platform }})" + include: + - platform: ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-windows-2022-x86-64-16x32' || 'windows-latest' }} + python: "3.14" + - platform: ${{ github.repository == 'astral-sh/ruff' && 'namespace-profile-macos-15' || 'macos-latest' }} + python: "3.13" + name: "cargo test (${{ matrix.platform }}, python ${{ matrix.python }})" runs-on: ${{ matrix.platform }} needs: determine_changes if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-test') && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} @@ -519,8 +625,21 @@ jobs: with: version: "0.12.3" enable-cache: "true" + # `PATH` as well as `PYTHON`, for the fallback reason described on + # `cargo test (python …)`. the interpreter reports its own directory rather than + # `dirname` doing it: on windows this shell is git bash but `uv python find` answers + # with a backslash-separated path, which `dirname` reads as a single component + - name: "Pin the interpreter the python-executing tests use" + env: + PY: ${{ matrix.python }} + run: | + uv python install "$PY" + python_path="$(uv python find "$PY")" + echo "PYTHON=$python_path" >> "$GITHUB_ENV" + "$python_path" -c 'import os, sys; print(os.path.dirname(sys.executable))' >> "$GITHUB_PATH" - name: "Run tests" run: | + python3 -VV cargo nextest run --all-features --profile ci cargo test --all-features --doc @@ -1481,6 +1600,7 @@ jobs: - cargo-fmt - cargo-clippy - cargo-test-linux + - cargo-test-interpreter - cargo-test-wasm - cargo-build-msrv - shellcheck diff --git a/Cargo.lock b/Cargo.lock index ba1fb6ac0e..d0e589bbb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -370,6 +370,7 @@ dependencies = [ "by_ir", "ruff_db", "ruff_python_ast", + "ruff_python_parser", "ruff_python_stdlib", "ruff_text_size", "thin-vec", diff --git a/docs/basedpython/cli-reference.md b/docs/basedpython/cli-reference.md index 5fe04e062b..9d5f748813 100644 --- a/docs/basedpython/cli-reference.md +++ b/docs/basedpython/cli-reference.md @@ -105,6 +105,12 @@ by compile --no-any # refuse to leave a gradual-typed function inter by compile --require-native # refuse to leave *any* function interpreted ``` +the output directory mirrors the *module* tree, the way `by build`'s does: the +package member `pkg/sub/dup.py` lands at `out/pkg/sub/dup.cpython-313-darwin.so`, +and the package `pkg/sub/__init__.py` at +`out/pkg/sub/__init__.cpython-313-darwin.so` — so `out/` can go on `sys.path` as +it stands and every module imports under the dotted name it was compiled as + `--no-any` buys no speed on its own — it is a **predictability contract**. a gradual type is the commonest reason a function silently stays interpreted, and a decline is invisible unless you look for it, so a module that means to be fully diff --git a/docs/basedpython/development/compilation/benchmarks.md b/docs/basedpython/development/compilation/benchmarks.md new file mode 100644 index 0000000000..dc2185c9a9 --- /dev/null +++ b/docs/basedpython/development/compilation/benchmarks.md @@ -0,0 +1,203 @@ +# the benchmark suite + +```sh +cargo build --release --bin by +uv run --no-project --python 3.13 python scripts/native-bench/bench.py +``` + +the suite lives in `scripts/native-bench/`: `bench.py` stages and builds, +`timer.py` times, `programs/` holds the benchmarks and `programs.toml` says what +each is for and what it is allowed to leave interpreted + +it needs a **release** build — a debug `by` is too slow to measure anything with +— and it needs `uv`, which fetches both the interpreter and mypyc + +## the one thing to know + +**the ratio is the measurement.** absolute times on this suite move by a factor +of four with machine load: cpython's own `dot` has read 47ms and 209ms on the +same laptop in the same week. no number here is reported from one build's clock +alone + +and **every row carries its own error bar**. each benchmark is compiled *twice*, +independently, by the same compiler, and both builds are timed. two builds of one +program are one program — literally: the generated C is byte-identical once the +module name is normalised out — so whatever ratio the suite reports between them +is noise it invented. that is the `noise` column, and nothing smaller than a +row's own noise is a finding + +## how a run works + +four builds of each benchmark, staged under four module names of their own: + +| build | what it is | +| --------- | ------------------------------ | +| `cpython` | the source, interpreted | +| `by` | `by compile` | +| `control` | `by compile` again, separately | +| `mypyc` | mypyc, for scale | + +the distinct names are what makes the method possible. an extension module's +init hook is found by name, so two of them cannot answer to `mandel` — under +`mandel_by` and `mandel_control` they can, and all four builds then live in one +process at once + +and because they live in one process they can be **interleaved**. a run is a +sequence of rounds, and in each round every build is timed once, in an order +that rotates so none of them is always the one paying for a cold cache. a load +spike therefore lands inside the same round as everything else, and mostly +cancels when the round is turned into a quotient + +## the statistics + +the **median**, throughout — never the mean, never the minimum + +- the mean is moved by the one round that hit a scheduler +- the minimum is an extreme-value statistic. its expectation depends on how many + rounds were run and on how quiet the machine happened to be, so a minimum is + not comparable with another minimum, which is the only thing this suite is for + +a ratio is **paired**: it is the median of the per-round quotients, not the +quotient of the two medians. dividing one build's median by another's would let +a spike that landed on only one of them through unchallenged + +the interval on each ratio is the distribution-free one for a median, taken from +the binomial tail. it is exact, assumes nothing about the shape of the noise, +needs no resampling and no random numbers — so two readings of the same data +agree. below nine rounds that interval degenerates to the range of everything +seen, so nine is a floor the harness enforces rather than a default + +one sample is many calls, and how many is calibrated per benchmark: enough that +the fastest build's sample is well clear of the clock, few enough that the +slowest build's round does not dominate the run. every build runs the same +number, so the pairing stays exact + +## what it refuses + +each of these is a way this suite has actually produced a wrong number, and each +is now a refusal rather than a warning: + +| refusal | the failure it closes | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| the artefact must be a real extension module | a broken shim made `by compile` fail silently and the previous `.so` was timed in its place | +| …under this run's own root, newer than this build | the run's root is a fresh temporary directory, so a stale one cannot be reached at all | +| an interpreted build must **not** be an extension | the mirror image, which would make the baseline a compiled one | +| every build must return cpython's answer | a build that got a different answer is not a faster build | +| the decline count must match `programs.toml` exactly | a benchmark can decline and quietly measure interpreted code while still posting a number | +| an unknown name, an unlisted program, an empty run | a harness that matches nothing looks exactly like a harness that found nothing wrong | + +the decline check is the one worth dwelling on. `by compile` never fails on code +it cannot lower — that function runs from its interpreted definition instead — so +a "compiled" number can silently be an interpreted one. `programs.toml` records +the expected count per benchmark and the run fails when it moves **in either +direction**: a compiler that started compiling something is as much a change as +one that stopped. improving the compiler edits that file in the same commit, +which makes it a readable ledger of what the backend has learned to take + +the refusals are exercised rather than asserted: + +```sh +uv run --no-project --python 3.13 python scripts/native-bench/bench.py --self-check +``` + +which builds a leg that is wrong in each of those ways and proves each one is +turned away + +## what it measures + +one axis per group, and a benchmark earns its place by being the only one on its +axis — or by being half of a pair whose *difference* is the axis: + +| group | benchmarks | the question | +| ---------- | ---------------------------------------- | ---------------------------------------------------- | +| `float` | `mandel`, `mandel_inline` | scalar float work; the pair isolates calls | +| `int` | `loops`, `bigint` | tagged integers, and leaving the word | +| `dispatch` | `calls`, `methods`, `recurse`, `inherit` | call, method, depth, and an override | +| `memory` | `alloc`, `fields`, `objects` | allocation and field access, apart and together | +| `list` | `dot`, `prefix`, `sieve` | indexed reads, growth, indexed writes | +| `tuple` | `tuples` | pack, unpack, and a two-value return | +| `set` | `sets` | membership as the whole operation | +| `dict` | `dictget`, `dicthist` | lookup-and-update, and the histogram miss | +| `str` | `words`, `chars`, `strops`, `keybuild` | build, scan, methods, and key construction | +| `boxing` | `generic`, `generic_mono` | the pair isolates what a type parameter costs | +| `control` | `excs` | a raise that is caught, and a `try` that never fires | +| `frames` | `gen`, `coro` | the two halves of the resumable-frame lowering | + +the pairs are read as pairs. `generic` alone says almost nothing; against +`generic_mono`, which is the same program with the call monomorphised by hand, +it says exactly what the type parameter costs. same for `mandel` against +`mandel_inline`, and for `objects` against `alloc`, `fields` and `methods` + +adding one is three files: the program, an entry in `programs.toml`, and a line +in the table above. the entry has to say what it measures that nothing else does + +## comparing two runs + +```sh +… bench.py --json today.json +… bench.py --baseline today.json +``` + +the quantity compared between runs is the **speedup**, not the time. absolute +times are not comparable across machines, across interpreters, or across a +Tuesday, and this suite exists because they were compared anyway + +a change is called one only when it clears the bar, and the bar is whichever is +larger: the noise the two runs measured for themselves, or +`--regression-threshold`. a regression exits non-zero, so it is detected rather +than eyeballed. a baseline from a different interpreter or a different host is +compared anyway but says so first, and a row either run found too noisy is +skipped rather than given a very wide bar + +**the noise column does not bound this.** the control is two builds in one +process at one moment, so it bounds the noise *within* a run. between two runs +there is more: a different process, a different heap layout, a different machine +mood. measured rather than assumed — two runs of an **unchanged** compiler, +forty minutes apart, agreed within 3% on 25 of 27 rows and disagreed by 8.1% and +8.4% on the other two, while both of those rows reported a ±0.8% floor for +themselves. so the default bar is 10%, which is what that evidence supports, and +not the within-run floor, which would have called both of them improvements + +the consequence is worth stating plainly: on a machine doing other work this +suite sees a 10% change and does not see a 5% one. for smaller than that, use a +machine that does nothing else, drop `--regression-threshold` to match the floor +it then reports, and confirm anything it flags by running it again + +## the half that can run anywhere + +```sh +… bench.py --verify-only +… bench.py --self-check +``` + +`--verify-only` does everything the suite does except look at a clock: it builds +all four ways, proves each artefact, checks that every build gives cpython's +answer, and checks the decline ledger. none of that depends on how busy the +machine is, so it is the half that belongs on a shared runner — and it is where +most of the *correctness* value is. it catches a benchmark that stopped +compiling, a compiled build that started answering differently, and a compiler +that quietly began declining something it used to take + +the timing half wants a machine of its own. a shared runner cannot hold a 3% +noise floor, and this suite's whole premise is that a number it cannot stand +behind should not be printed as if it could. run the table on a fixed machine, +keep the `--json`, and gate on `--baseline` against the previous one + +## reading a bad run + +a row whose noise exceeds `--noise-limit` (10% by default) is marked `!` and +does not count as measured. the run then exits non-zero even if everything else +looks fine, because a table that is *mostly* trustworthy is the kind nobody +remembers to check before quoting one of its rows + +that limit is not fastidiousness. pairing cancels the *noise* a busy machine +adds, but it does not cancel the **bias**: under preemption the longer sample +loses more, and the interpreted build's sample is the long one, so contention +pushes the reported speedup *up*. a run at load 95 on a 16-core laptop read +`fields` at 23.8x where two quiet runs both put it near 12x, and the two +supposedly identical builds disagreed by 74% in the same breath. widening the +bar does not fix a bias, so a row that noisy is skipped rather than compared + +between those two extremes the floor is simply reported. a `±6%` floor means +the run can see a 2x difference and cannot see a 10% one, which is often all +that was wanted diff --git a/docs/basedpython/development/compilation/index.md b/docs/basedpython/development/compilation/index.md index 36513b3d30..b5803f0542 100644 --- a/docs/basedpython/development/compilation/index.md +++ b/docs/basedpython/development/compilation/index.md @@ -173,4 +173,6 @@ because two of them change this design rather than extend it: what each buys the compiler, and what codegen needs their design to preserve - [runtime](runtime.md) — object model, refcounting, exceptions, interop, debugging +- [benchmarks](benchmarks.md) — what the suite measures, the method it enforces, + and what it refuses to time - [plan](plan.md) — semantic deltas, testing, milestones, risks diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index 6187612e46..061f96dbb2 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -124,6 +124,38 @@ a class whose construction fell back to the interpreted definition is skipped: t `def`s the fallback ran already carry their decorators, and applying them a second time would wrap twice +### how many times a decorator runs + +python evaluates a decorator **once**, where the definition stands. the twin is +what stands there, so the twin evaluates it — and module init then evaluates the +same decorator a second time over the native definition that replaced the twin's. +the name each definition ends up bound to is right either way, so nothing shows +but the side effect: `@register` puts two entries in its registry, `@count_them` +counts one function twice + +for a module-level **function** and a module-level **class** the decorator is +therefore blanked out of the source the twin runs, so init's is the only +evaluation. blanking rather than cutting keeps every line where it was, which is +what a traceback through the twin quotes + +that leaves a window: from the twin's `def` or `class` to the moment init reaches +it, the name holds a definition nothing has decorated yet. only the module's own +body can look — everything else runs after init — so a definition whose name that +body reads **declines** rather than be compiled and decorated twice. the reads +followed are the ones the body makes as it runs, plus, transitively, everything +held behind any definition it names: `TABLE = f()` reads directly, +`def g(): return f()` called at import reads just the same. an annotation counts +only where python evaluates one, so a module with +`from __future__ import annotations` may name a class in a signature freely + +a **method's** decorator still runs twice. it is not only a side effect that +would move: the class construction itself reads what the decorator wrote, and +`ABCMeta` is the case — it computes `__abstractmethods__` from the namespace the +body left, so taking `@abstractmethod` out of the twin empties that set on every +class whose construction falls back to the interpreted definition. the answer +there is to carry the decorated method *across* from the twin rather than to +re-apply the decorator, which is not built + ### boxed classes and interpreted fallbacks a construct with no native lowering is not a compile error. the module emits its diff --git a/pyproject.toml b/pyproject.toml index db5f15cdf6..9a5fa921d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,11 @@ release = { requires-python = ">=3.12" } [tool.ruff] target-version = "py38" extend-exclude = [ + # benchmark programs are measurement inputs, not code. an autofix that + # rewrites `i = i + 1` or hoists a loop changes what is being measured, and + # the isort rule alone would put `from __future__ import annotations` at the + # top of every one of them + "scripts/native-bench/programs/", "crates/ty_vendored/vendor/", "crates/ruff/resources/", "crates/ruff_linter/resources/", diff --git a/scripts/native-bench/bench.py b/scripts/native-bench/bench.py index 884b2e4dcb..b9404376e4 100644 --- a/scripts/native-bench/bench.py +++ b/scripts/native-bench/bench.py @@ -145,6 +145,17 @@ def as_json(self) -> dict: return {"median": self.median, "low": self.low, "high": self.high} +def ratio_json(result: Result, leg: str) -> dict | None: + """a leg's paired ratio as json, or `None` where there was no pairing + + the three json rows all read a ratio that may not exist — a leg the run did not + time has none — and two of the three used to read it without asking. one place + to forget rather than three + """ + ratio = result.ratio(leg) + return ratio.as_json() if ratio else None + + def paired(numerator: list[float], denominator: list[float]) -> Ratio: """the ratio of two builds, round by round rather than time against time @@ -537,9 +548,7 @@ def run_program( # ── reporting ──────────────────────────────────────────────────────────────── -def render( - results: list[Result], metadata: dict, show_declines: bool, limit: float -) -> None: +def render(results: list[Result], metadata: dict, show_declines: bool, limit: float): header = ( f"{'benchmark':<15}{'group':<10}{'cpython':>10}{'by':>10}{'mypyc':>10}" f" {'vs cpython':>15}{'vs mypyc':>16}{'noise':>10}{'dec':>5}" @@ -571,6 +580,13 @@ def render( against_cpython = result.ratio("cpython") against_mypyc = result.ratio("mypyc") noise = result.noise + # `ok` says every leg ran, not that every leg was *timed* — a leg whose samples + # were all discarded leaves no median behind. rather than assume the status + # covers it, say so in the row: a blank number is a measurement nobody has, + # which is not the same as a slow one + if cpython is None or by is None or noise is None: + print(f"{name:<15}{group:<10} no timing recorded") + continue print( f"{name:<15}{group:<10}" f"{cpython * 1000:>9.2f}m{by * 1000:>9.2f}m" @@ -614,7 +630,7 @@ def render( print(f" {decline}") -def render_verification(results: list[Result]) -> None: +def render_verification(results: list[Result]): """everything the suite checks that does not involve a clock this half is deterministic, so it is the half that can run anywhere — a @@ -656,10 +672,17 @@ def compare( previous = baseline["benchmarks"].get(result.program.name) if result.status != "ok" or previous is None or previous.get("status") != "ok": continue + against_cpython = result.ratio("cpython") + new_noise = result.noise + # the same gap as in the table above: `ok` does not promise a timing survived. + # a row with nothing to compare is left out of the comparison rather than + # compared against a number that is not there + if against_cpython is None or new_noise is None: + continue old = previous["vs_cpython"]["median"] - new = result.ratio("cpython").median + new = against_cpython.median change = new / old - 1 - old_noise, new_noise = previous["noise"], result.noise + old_noise = previous["noise"] # a row either run could not measure is skipped rather than compared # with a very wide bar. a pair of identical builds that disagreed by 70% # says the machine was preempting samples, and under preemption the @@ -706,7 +729,7 @@ def self_check(by: Path, python: str, python_version: str) -> int: root = Path(tempfile.mkdtemp(prefix="native-bench-selfcheck-")) failures = [] - def expect(what: str, refusal: str | None, wanted: str) -> None: + def expect(what: str, refusal: str | None, wanted: str): if refusal is None: failures.append(f"{what}: was accepted, and should have been refused") elif wanted not in refusal: @@ -897,6 +920,9 @@ def main() -> int: "confidence interval degenerates to the range of what was seen" ) + # self-check returns before anything reads this, but an empty list rather than an + # unbound name says that here instead of leaving it to be inferred from control flow + programs: list[Program] = [] if not args.self_check: programs = load_manifest(args.programs) @@ -929,7 +955,15 @@ def main() -> int: ) if probe.returncode != 0: raise Failure(f"{python} does not run: {probe.stderr.strip()}") - full_version, implementation, _ = probe.stdout.split() + # the probe below prints three fields, but a python that printed something else + # would unpack into an unhelpful ValueError here rather than say what it answered + fields = probe.stdout.split() + if len(fields) < 2: + raise Failure( + f"{python} answered {probe.stdout.strip()!r}, " + "not a version and an implementation" + ) + full_version, implementation = fields[0], fields[1] python_version = ".".join(full_version.split(".")[:2]) if args.self_check: @@ -1019,11 +1053,9 @@ def main() -> int: "times": result.times, **( { - "vs_cpython": result.ratio("cpython").as_json(), - "vs_mypyc": result.ratio("mypyc").as_json() - if result.ratio("mypyc") - else None, - "control": result.ratio("control").as_json(), + "vs_cpython": ratio_json(result, "cpython"), + "vs_mypyc": ratio_json(result, "mypyc"), + "control": ratio_json(result, "control"), "noise": result.noise, "noisy": result.noisy(args.noise_limit), } diff --git a/scripts/native-bench/programs/alloc.py b/scripts/native-bench/programs/alloc.py index 6bcdd89d4f..dedc6111b3 100644 --- a/scripts/native-bench/programs/alloc.py +++ b/scripts/native-bench/programs/alloc.py @@ -8,7 +8,7 @@ class Pair: - def __init__(self, x: int, y: int) -> None: + def __init__(self, x: int, y: int): self.x = x self.y = y diff --git a/scripts/native-bench/programs/fields.py b/scripts/native-bench/programs/fields.py index 37c166e071..d908af06a3 100644 --- a/scripts/native-bench/programs/fields.py +++ b/scripts/native-bench/programs/fields.py @@ -7,7 +7,7 @@ class State: - def __init__(self) -> None: + def __init__(self): self.a = 0 self.b = 1 self.c = 2 diff --git a/scripts/native-bench/programs/inherit.py b/scripts/native-bench/programs/inherit.py index 16bb3a936d..dca699b8e7 100644 --- a/scripts/native-bench/programs/inherit.py +++ b/scripts/native-bench/programs/inherit.py @@ -8,7 +8,7 @@ class Shape: - def __init__(self, size: int) -> None: + def __init__(self, size: int): self.size = size def area(self) -> int: diff --git a/scripts/native-bench/programs/methods.py b/scripts/native-bench/programs/methods.py index f1cd72e7f7..c6b6b24c4f 100644 --- a/scripts/native-bench/programs/methods.py +++ b/scripts/native-bench/programs/methods.py @@ -8,7 +8,7 @@ class Counter: - def __init__(self, base: int) -> None: + def __init__(self, base: int): self.base = base def step(self, k: int) -> int: diff --git a/scripts/native-bench/programs/objects.py b/scripts/native-bench/programs/objects.py index 1ab45e52e3..d1ccf5d27f 100644 --- a/scripts/native-bench/programs/objects.py +++ b/scripts/native-bench/programs/objects.py @@ -7,7 +7,7 @@ class Vec: - def __init__(self, x: float, y: float) -> None: + def __init__(self, x: float, y: float): self.x = x self.y = y diff --git a/zensical.toml b/zensical.toml index 51d0b20cfc..4dbab0b31a 100644 --- a/zensical.toml +++ b/zensical.toml @@ -220,6 +220,7 @@ development = [ "development/compilation/optimizations.md", "development/compilation/planned-features.md", "development/compilation/runtime.md", + "development/compilation/benchmarks.md", "development/compilation/plan.md", ] }, ] From 7d10c117a3d6e6051f62297755e468c6c31da2a0 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:50:46 +1000 Subject: [PATCH 7/7] a parameter its own body rebinds stands for no one value, so it keeps no bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `body_parameter_constraints` already computes `single_bindings` — a name bound more than once cannot stand for one value — and applied it to every local except the parameter itself. this applies it there too, which is the rule the file already states rather than an exception carved for this case. measured over the 153 top-level stdlib modules, two binaries from one tree: 5041 diagnostics to 4790. **−254 by location and +3**, across 41 modules. the other 46 of the 49 added lines are the same diagnostic at the same file:line:col with a differently rendered type, and read better for it — `has no attribute 'startswith'` on `object` becomes `not defined on 'None' in union 'Unknown | str | None'`, which names the actual bug in `inspect.getsourcefile`'s None return. all 3 genuinely new ones are correct. of the 254 removed, about 249 are false positives and 5 are real. the strongest thing lost is `code._showtraceback`, where `sys.exc_info()` gives `BaseException | None` against a body calling `value.with_traceback(tb)` — and that only survived because the same body's rebinding was not reachable above it. ⚠️ this masks rather than fixes. two of the three false-positive families it removes have nothing to do with rebinding: a recovered protocol records no `__getitem__`, no operator dunders and no `__iter__`, and a member called twice is pinned to the first call's argument types. both still fire on a parameter nobody rebinds. a third family is genuinely caused by rebinding — a rebind inside a loop retroactively hides a requirement at the loop head, so `ast.visit_If` ends up contradicting its own recovered signature above the line that rebinds. the narrower rule — drop the bound only when the rebinding reads a member off the name — was built and measured: 109 removals, a strict subset, and it loses the two `code.py` signals anyway (`value = value.with_traceback(tb)` is that shape) while keeping ~145 of the false positives. strictly worse on every axis. --- .../mdtest/basedpython_sound_types.md | 44 +++++++++++++++++++ .../src/types/inferred_signature.rs | 31 +++++++++---- docs/basedpython/features/sound-types.md | 13 ++++++ 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md index bc293d18f0..0a12fe5103 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md @@ -567,6 +567,50 @@ def f(x): f("anything") # ok ``` +### a parameter its own body rebinds says nothing + +a name bound more than once cannot stand for one value, which is already why a reassigned local +contributes nothing. a parameter is no different once its own body rebinds it: below the rebinding +the name is whatever the rebinding produced, so what is done with it there requires nothing of what +the caller passed + +keeping the requirements collected *above* the rebinding would not do either. walking a traceback +requires only that the argument have a `tb_next`, so the rebinding lands on that member's type — +which is `object`, because nothing said what it holds — and the read below it then fails against the +bound the function itself produced + +```py +def deepest(tb): + if tb.tb_next: + tb = tb.tb_next + return tb.tb_frame + +deepest("anything") # ok +``` + +the same holds when the member is a method, which is the shape most of these take + +```py +def rebound(x): + x.foo() + x = x.foo() + x.foo() + +rebound("anything") # ok +``` + +the rebinding does not have to be reachable, or to come from the parameter, for the name to stop +standing for one value + +```py +def maybe(x, flag): + x.foo() + if flag: + x = 1 + +maybe("anything", True) # ok +``` + ### a recursive call does not constrain ```py diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index e145b5a8aa..d5c8a1ad58 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -393,8 +393,8 @@ pub(crate) fn body_parameter_constraints<'db>( // a name bound more than once cannot stand for one value: which of them a later use is // about is not a question this can answer - let single_bindings = index - .place_table(body_scope.file_scope_id(db)) + let place_table = index.place_table(body_scope.file_scope_id(db)); + let single_bindings: FxHashSet = place_table .symbols() .filter(|symbol| symbol.is_bound() && !symbol.is_reassigned()) .map(|symbol| symbol.name().clone()) @@ -431,13 +431,26 @@ pub(crate) fn body_parameter_constraints<'db>( entries.extend(asserted_parameter_types(db, env, index, body_scope, node)); // a parameter a nested scope captured keeps nothing: that body is checked against this - // bound, and this walk never saw what it does with the name - if !collector.captured.is_empty() { - entries.retain(|(parameter, _)| { - parameter_definition_name(db, *parameter) - .is_none_or(|name| !collector.captured.contains(&name)) - }); - } + // bound, and this walk never saw what it does with the name. + // + // a parameter its own body rebinds keeps nothing either, for the same reason a rebound + // local does. after + // + // while tb.tb_next: + // tb = tb.tb_next + // + // the name stands for whatever the rebinding produced, not for what the caller passed, so + // the uses below it are no requirement on the argument — and bounding the argument by them + // anyway makes the body fail against its own bound, because the rebinding lands on the + // member type the bound itself invented + entries.retain(|(parameter, _)| { + parameter_definition_name(db, *parameter).is_none_or(|name| { + !collector.captured.contains(&name) + && place_table + .symbol_id(name.as_str()) + .is_some_and(|symbol| !place_table.symbol(symbol).is_reassigned()) + }) + }); entries.sort_by_key(|(parameter, _)| *parameter); diff --git a/docs/basedpython/features/sound-types.md b/docs/basedpython/features/sound-types.md index 57d874a62d..200db76001 100644 --- a/docs/basedpython/features/sound-types.md +++ b/docs/basedpython/features/sound-types.md @@ -174,6 +174,19 @@ that a member's value was given to is read the same way: a name bound more than for one value, and a use under a narrowing is about something narrower than the value it was bound to +a parameter its own body rebinds keeps nothing at all, for the same reason. the reads above the +rebinding are not enough on their own: walking a linked structure asks only that the argument have +the member it walks along, so the rebinding lands on that member — whose value nothing described — +and the walk's next step would fail against the signature the function itself produced + +```python +def deepest(tb): + if tb.tb_next: + tb = tb.tb_next + return tb.tb_frame +# def deepest(tb) +``` + ### an `assert` at the top of the body an `assert` there holds for every call that returns normally, so it is the author saying what they