From dd1d764d8c679293931c9ff09d4d7010353e5fab Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+kotlinisland@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:29:51 +1000 Subject: [PATCH] lower the match statement and runtime unions for older pythons, and report what neither covers --- crates/by_build/src/lib.rs | 6 +- crates/by_build/tests/differential.rs | 13 +- crates/by_build/tests/end_to_end.rs | 11 +- crates/by_transforms/src/lib.rs | 44 +- .../src/transforms/annotation.rs | 116 ++- .../src/transforms/ast_driver.rs | 22 +- .../src/transforms/dedent_string.rs | 14 +- .../src/transforms/destructure.rs | 60 +- crates/by_transforms/src/transforms/if_let.rs | 59 +- .../src/transforms/kw_subscript.rs | 23 +- .../src/transforms/match_polyfill.rs | 863 ++++++++++++++++++ crates/by_transforms/src/transforms/mod.rs | 2 + .../src/transforms/mutable_defaults.rs | 14 +- .../src/transforms/optional_type.rs | 91 +- .../src/transforms/runtime_union.rs | 343 +++++++ crates/by_transforms/src/type_info.rs | 24 + crates/ty/src/by_commands.rs | 2 +- crates/ty/tests/by_e2e.rs | 9 +- .../development/how-transpilation-works.md | 20 +- docs/basedpython/features/polyfills.md | 68 +- 20 files changed, 1635 insertions(+), 169 deletions(-) create mode 100644 crates/by_transforms/src/transforms/match_polyfill.rs create mode 100644 crates/by_transforms/src/transforms/runtime_union.rs diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index 3e47e542b7..8af73006c0 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -76,8 +76,9 @@ pub fn emit_lowered( source: &str, out_dir: &Path, options: &Options, + version: Option<(u8, u8)>, ) -> Result { - let module = finish(module, source, options, None)?; + let module = finish(module, source, options, version)?; emit_verified(&module, out_dir, options) } @@ -90,8 +91,9 @@ pub fn emit_source( module_name: impl Into, out_dir: &Path, options: &Options, + version: Option<(u8, u8)>, ) -> Result { - let module = lower(source, module_name, options, None)?; + let module = lower(source, module_name, options, version)?; emit_verified(&module, out_dir, options) } diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index fa6996afcf..138a9c46b4 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -540,11 +540,18 @@ fn agree_in( let module = format!("by_diff_{tag}"); // the interpreted leg: for basedpython, the transpiler's own output run by - // cpython, under the same config `by_build` uses, so the two legs are the same - // program. for python there is nothing to transpile — it already is one + // cpython, under the same config `by_build` uses — including the *target + // version*, which has to be this interpreter's or the two legs are not the + // same program. for python there is nothing to transpile — it already is one let interpreted_source = match language { by_irbuild::Language::BasedPython => { - by_transforms::transpile(source, &Config::default()).expect("the source transpiles") + let mut config = Config::default(); + if let Some((major, minor)) = toolchain.version + && let Ok(parsed) = format!("{major}.{minor}").parse() + { + config.min_version = parsed; + } + by_transforms::transpile(source, &config).expect("the source transpiles") } by_irbuild::Language::Python => source.to_string(), }; diff --git a/crates/by_build/tests/end_to_end.rs b/crates/by_build/tests/end_to_end.rs index 18ccc3e96a..f304da365f 100644 --- a/crates/by_build/tests/end_to_end.rs +++ b/crates/by_build/tests/end_to_end.rs @@ -570,6 +570,7 @@ def kept(a: str, b: str) -> object: "by_e2e_append", &dir, &Options::default(), + None, ) .expect("the module emits"); let emitted = std::fs::read_to_string(&built.artifact.source).expect("the C is readable"); @@ -1097,8 +1098,14 @@ fn the_emitted_c_names_no_pointer_type_it_does_not_mean() { } let dir = std::env::temp_dir().join("by_e2e_pointers"); let _ = std::fs::remove_dir_all(&dir); - let built = by_build::emit_source(POINTER_SOURCE, "by_e2e_pointers", &dir, &Options::default()) - .expect("the module emits"); + let built = by_build::emit_source( + POINTER_SOURCE, + "by_e2e_pointers", + &dir, + &Options::default(), + None, + ) + .expect("the module emits"); let object = dir.join("by_e2e_pointers.o"); let mut args = by_build::compile_command(&toolchain, &built.artifact.source, &object, &dir); diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 8fd7792290..68ba0e6ee7 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -201,9 +201,11 @@ pub fn transpile(source: &str, config: &Config) -> Result { let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &model.eagerly_imported_modules()); + let final_output = run_version_polyfill_phase(final_output, config); // --- Phase 3: syntax verification --- verify_syntax(&final_output).map_err(|e| e.message)?; + verify_target_syntax(&final_output, config).map_err(|e| e.message)?; Ok(final_output) } @@ -365,6 +367,7 @@ pub fn transpile_typed_with_map( let final_output = run_import_redirect_phase(output, config); let final_output = run_anon_named_tuple_cleanup(final_output, config)?; let final_output = run_lazy_import_phase(final_output, config, &eager_imports); + let final_output = run_version_polyfill_phase(final_output, config); // phases 1-2c only prepend preambles at the top and edit within lines, so // the spliced body keeps its line correspondence: prepend one `None` per @@ -398,7 +401,9 @@ pub fn transpile_typed_with_map( line_map.extend(composed[kept..].iter().copied()); // verify last: on failure, map the generated span back to a `.by` range - if let Err(mut err) = verify_syntax(&final_output) { + let verified = + verify_syntax(&final_output).and_then(|()| verify_target_syntax(&final_output, config)); + if let Err(mut err) = verified { err.by_range = err.output_range.and_then(|r| { output_offset_to_by_range(&line_map, &final_output, original_source, r.start()) }); @@ -571,6 +576,43 @@ fn run_lazy_import_phase(source: String, config: &Config, eager: &[String]) -> S } } +/// Version polyfill phase: rewrite syntax the target python cannot parse into +/// syntax it can. Runs over the finished python rather than over `.by`, so a +/// `match` an earlier lowering *generated* — for a `let` destructuring, an +/// `if let`, a statement expression — is lowered by the same code as one the +/// author wrote. +/// +/// Nothing here changes how many lines the file has: the polyfill rewrites +/// headers in place and pads them back to their original height, so the only +/// lines it adds are the runtime preamble's, at the top, where the line map +/// already accounts for generated leading lines. +fn run_version_polyfill_phase(source: String, config: &Config) -> String { + transforms::match_polyfill::lower(source, config.min_version) +} + +/// Re-parse the transpiled output *as the target python version* and report any +/// construct that version cannot parse. +/// +/// [`verify_syntax`] asks whether the output is python at all; this asks whether +/// it is python the file's declared floor can run. Without it, syntax no +/// polyfill covers — `except*`, t-strings, a PEP 701 f-string — is emitted +/// verbatim and fails at import time in generated code the author never wrote. +fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileError> { + let options = ruff_python_parser::ParseOptions::from(ruff_python_ast::PySourceType::Python) + .with_target_version(config.min_version); + let parsed = ruff_python_parser::parse_unchecked(source, options); + let Some(first) = parsed.unsupported_syntax_errors().first() else { + return Ok(()); + }; + // the parser's own wording already names both versions: "Cannot use + // `except*` on Python 3.9 (syntax was added in Python 3.11)" + Err(TranspileError { + message: first.to_string(), + output_range: Some(first.range), + by_range: None, + }) +} + /// Splice `preamble` into `body` where generated lines belong: after the module /// docstring and any `from __future__ import`, each of which is only valid /// where it already is. diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index 2e8147c283..7039f07bac 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -1,12 +1,23 @@ use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::{Expr, Stmt}; +use ruff_python_ast::{Expr, PythonVersion, Stmt}; use ruff_text_size::Ranged; +use crate::Config; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::transforms::{literal_types, optional_type}; use crate::type_info::TypeInfo; +/// The element type an unpacked tuple element wraps, whichever way the target +/// spells the unpack. +fn strip_unpack(element: &str) -> Option<&str> { + element.strip_prefix('*').or_else(|| { + element + .strip_prefix("Unpack[") + .and_then(|rest| rest.strip_suffix(']')) + }) +} + /// Rewrites tuple literal types in type positions. /// /// `a: (int, str)` → `a: tuple[int, str]` @@ -18,18 +29,40 @@ use crate::type_info::TypeInfo; pub(crate) struct TupleLiteralType<'src> { source: &'src str, types: &'src dyn TypeInfo, + min_version: PythonVersion, + /// set when a lowering spelled an `Unpack`, so the pass can ask for the import + pub(crate) needs_unpack_import: std::cell::Cell, pub(crate) edits: Vec, } impl<'src> TupleLiteralType<'src> { - pub(crate) fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self { + pub(crate) fn new( + source: &'src str, + types: &'src dyn TypeInfo, + min_version: PythonVersion, + ) -> Self { Self { source, types, + min_version, + needs_unpack_import: std::cell::Cell::new(false), edits: Vec::new(), } } + /// How an unpacked element of a tuple type is spelled for the target. The + /// star form is PEP 646, python 3.11; below that it is a *syntax* error + /// rather than something the `__future__` import could defer, so a tuple + /// type has to say `Unpack[...]` instead. + fn unpack(&self, inner: &str) -> String { + if self.min_version >= PythonVersion::PY311 { + format!("*{inner}") + } else { + self.needs_unpack_import.set(true); + format!("Unpack[{inner}]") + } + } + fn src(&self, range: ruff_text_size::TextRange) -> &str { &self.source[usize::from(range.start())..usize::from(range.end())] } @@ -75,7 +108,7 @@ impl<'src> TupleLiteralType<'src> { // rather than the wrapped `tuple[*tuple[T, ...]]` form if parameter_shape && lowered.len() == 1 - && let Some(rest) = lowered[0].strip_prefix("*") + && let Some(rest) = strip_unpack(&lowered[0]) { return Some(rest.to_owned()); } @@ -218,7 +251,7 @@ impl<'src> TupleLiteralType<'src> { let value_src = self .transform_annotation(&named.value) .unwrap_or_else(|| self.src(named.value.range()).to_owned()); - return format!("*tuple[{value_src}, ...]"); + return self.unpack(&format!("tuple[{value_src}, ...]")); } self.transform_annotation(&named.value) .unwrap_or_else(|| self.src(named.value.range()).to_owned()) @@ -233,9 +266,9 @@ impl<'src> TupleLiteralType<'src> { .transform_annotation(&s.value) .unwrap_or_else(|| self.src(s.value.range()).to_owned()); if parameter_shape { - format!("*tuple[{value_src}, ...]") + self.unpack(&format!("tuple[{value_src}, ...]")) } else { - format!("*{value_src}") + self.unpack(&value_src) } } // a plain element type — also lower a nested `?` (`(int, str?)`), @@ -243,7 +276,7 @@ impl<'src> TupleLiteralType<'src> { // tuple's whole-expression edit subsumes the optional pass's edit _ => self .transform_annotation(elt) - .or_else(|| optional_type::rewrite_type_expr(self.source, elt)) + .or_else(|| optional_type::rewrite_type_expr(self.source, elt, self.min_version)) .unwrap_or_else(|| self.src(elt.range()).to_owned()), } } @@ -275,17 +308,18 @@ impl TypeExprVisitor for TupleLiteralType<'_> { pub(crate) struct TupleLiteralTypePass<'src> { source: &'src str, + config: Config, } impl<'src> TupleLiteralTypePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, config: Config) -> Self { + Self { source, config } } } impl TypeAwarePass for TupleLiteralTypePass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner = TupleLiteralType::new(self.source, types); + let mut inner = TupleLiteralType::new(self.source, types, self.config.min_version); walk_type_positions(stmts, Some(types), &mut inner); let mut wraps_literal = false; for fix in inner.edits { @@ -298,6 +332,10 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { ctx.text_edits.push((range, repl)); } } + if inner.needs_unpack_import.get() { + ctx.required_imports + .push("from typing import Unpack".to_owned()); + } // when our embedded literal-type lowering produced `Literal[...]` text, // request the import. the standalone literal_types pass doesn't see // the bare literal anymore because we've replaced its parent annotation @@ -311,7 +349,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { #[cfg(test)] mod tests { use crate::python_passthrough::unchanged; - use crate::{Config, transpile}; + use crate::{Config, PythonVersion, transpile}; use indoc::indoc; fn check(input: &str, expected: &str) { @@ -321,6 +359,18 @@ mod tests { ); } + /// the same, against a target that has PEP 646 + fn check_at(min_version: PythonVersion, input: &str, expected: &str) { + let config = Config { + min_version, + ..Config::test_default() + }; + assert_eq!( + transpile(input, &config).unwrap(), + crate::python_passthrough::lazify_expected(expected) + ); + } + #[test] fn simple_tuple_annotation() { check("a: (int, str)\n", "a: tuple[int, str]\n"); @@ -331,21 +381,39 @@ mod tests { check("a: (int,)\n", "a: tuple[int]\n"); } - /// a bare `*A` splices `A` in — python spells that the same way, unlike the - /// `*: T` variadic it shares an AST shape with + /// a bare `*A` splices `A` in — python spells that the same way from 3.11, + /// unlike the `*: T` variadic it shares an AST shape with #[test] fn unpacked_element() { - check("a: (int, *A)\n", "a: tuple[int, *A]\n"); + check_at( + PythonVersion::PY311, + "a: (int, *A)\n", + "a: tuple[int, *A]\n", + ); + } + + /// a star in a subscript is 3.11 grammar, so a `__future__` import cannot + /// defer it the way it defers an annotation's *evaluation* — below 3.11 the + /// unpack has to be spelled out + #[test] + fn unpacked_element_before_pep_646() { + check( + "a: (int, *A)\n", + indoc! {" + from typing_extensions import Unpack + a: tuple[int, Unpack[A]] + "}, + ); } #[test] fn lone_unpacked_element_without_a_comma() { - check("a: (*A)\n", "a: tuple[*A]\n"); + check_at(PythonVersion::PY311, "a: (*A)\n", "a: tuple[*A]\n"); } #[test] fn lone_unpacked_element_with_a_comma() { - check("a: (*A,)\n", "a: tuple[*A]\n"); + check_at(PythonVersion::PY311, "a: (*A,)\n", "a: tuple[*A]\n"); } #[test] @@ -458,7 +526,18 @@ mod tests { fn variadic_tuple_annotation() { // `*: T` in a tuple type expands to `*tuple[T, ...]` so the tuple // can hold zero+ values of T after the leading positional fields - check("b: (int, *: str)\n", "b: tuple[int, *tuple[str, ...]]\n"); + check_at( + PythonVersion::PY311, + "b: (int, *: str)\n", + "b: tuple[int, *tuple[str, ...]]\n", + ); + check( + "b: (int, *: str)\n", + indoc! {" + from typing_extensions import Unpack + b: tuple[int, Unpack[tuple[str, ...]]] + "}, + ); } #[test] @@ -466,7 +545,8 @@ mod tests { // `*name: T` in a tuple type behaves the same as `*: T` — the name // is metadata for callable-parameter use and has no effect on tuple // type semantics - check( + check_at( + PythonVersion::PY311, "b: (int, *args: str)\n", "b: tuple[int, *tuple[str, ...]]\n", ); diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 13cee5b368..d15b8652ba 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -44,9 +44,9 @@ use super::{ implicit_typing, inferred_annotation, init_method, just_float, kw_subscript, literal_string, literal_types, local_once, main_function, match_type, modifiers, mutable_defaults, none_chain, optional_type, overload, parametric_is, postfix_await, propagate, properties, protocol_type, - raises_clause, reified_generic, repeated_underscore, sentinel, some_ctor, soundness, - statement_expression, string_tag, super_keyword, symbolic_type_op, template_type, top_star, - trailing_lambda, tuple_index, type_fn, type_is, type_reification, typed_dict_literal, + raises_clause, reified_generic, repeated_underscore, runtime_union, sentinel, some_ctor, + soundness, statement_expression, string_tag, super_keyword, symbolic_type_op, template_type, + top_star, trailing_lambda, tuple_index, type_fn, type_is, type_reification, typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, unpack, use_site_variance, }; use crate::Config; @@ -546,7 +546,7 @@ pub(crate) fn run_against_source<'a>( let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new(source_ref); let just_float_pass = just_float::JustFloatPass::new(); let float_const_pass = float_const::FloatConstPass::new(); - let kw_subscript_pass = kw_subscript::KwSubscriptPass::new(source_ref); + let kw_subscript_pass = kw_subscript::KwSubscriptPass::new(source_ref, config.min_version); let generic_call_pass = generic_call::GenericCallStripPass::new(source_ref); let reified_generic_pass = reified_generic::ReifiedGenericPass::new(source_ref, config.min_version); @@ -556,7 +556,7 @@ pub(crate) fn run_against_source<'a>( let implicit_typing_pass = implicit_typing::ImplicitTypingPass::new(); let inferred_annotation_pass = inferred_annotation::InferredAnnotationPass::new(); let template_type_pass = template_type::TemplateTypePass; - let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref); + let tuple_types_pass = annotation::TupleLiteralTypePass::new(source_ref, config.clone()); let literal_types_pass = literal_types::LiteralTypePass::new(source_ref); let callable_pass = callable::CallableSyntaxPass::new(source_ref); let protocol_type_pass = protocol_type::ProtocolTypePass::new(source_ref, config.clone()); @@ -565,13 +565,14 @@ pub(crate) fn run_against_source<'a>( let some_ctor_pass = some_ctor::SomeCtorPass::new(); let propagate_pass = propagate::PropagatePass::new(source_ref); let none_chain_pass = none_chain::NoneChainPass::new(source_ref); - let optional_type_pass = optional_type::OptionalTypePass::new(source_ref); + let optional_type_pass = optional_type::OptionalTypePass::new(source_ref, config.min_version); + let runtime_union_pass = runtime_union::RuntimeUnionPass::new(config.min_version); let generics_pass = generics::GenericPolyfillPass::new(source_ref, config.clone()); let soundness_pass = soundness::SoundnessPass::new(source_ref, config); let checked_cast_pass = checked_cast::CheckedCastPass; let trailing_lambda_pass = trailing_lambda::TrailingLambdaPass::new(source_ref); - let if_let_pass = if_let::IfLetPass::new(source_ref, config.min_version); - let destructure_pass = destructure::DestructurePass::new(source_ref, config.min_version); + let if_let_pass = if_let::IfLetPass::new(source_ref); + let destructure_pass = destructure::DestructurePass::new(source_ref); let statement_expression_pass = statement_expression::StatementExpressionPass::new(source_ref); let context_params_pass = context_params::ContextParamsPass::new(source_ref); let extension_block_pass = extension::ExtensionBlockPass::new(source_ref); @@ -776,6 +777,11 @@ pub(crate) fn run_against_source<'a>( // `T?` → `T | None`; a type-position edit, disjoint from the // value-position `??` / `?.` lowerings below &optional_type_pass, + // a PEP 604 union the runtime will evaluate (`isinstance(x, int | str)`) + // is spelled the way the target can. its template covers the whole + // union, and the sort puts a wider replacement first, so the lowerings + // inside each arm are materialized rather than dropped + &runtime_union_pass, // coalesce sees `?.` LHS via source ranges; must run BEFORE // none_chain so its wider `??` edit wins over none_chain's narrow // `?.` edit when both target the same span diff --git a/crates/by_transforms/src/transforms/dedent_string.rs b/crates/by_transforms/src/transforms/dedent_string.rs index 5055278d6f..4132411cdc 100644 --- a/crates/by_transforms/src/transforms/dedent_string.rs +++ b/crates/by_transforms/src/transforms/dedent_string.rs @@ -138,8 +138,16 @@ mod tests { use indoc::indoc; fn check(input: &str, expected: &str) { + check_at(Config::test_default().min_version, input, expected); + } + + fn check_at(min_version: crate::PythonVersion, input: &str, expected: &str) { + let config = Config { + min_version, + ..Config::test_default() + }; assert_eq!( - transpile(input, &Config::test_default()).unwrap(), + transpile(input, &config).unwrap(), crate::python_passthrough::lazify_expected(expected) ); } @@ -182,9 +190,11 @@ mod tests { ); } + /// t-strings are 3.14 syntax, so the target has to be one that can run them #[test] fn tstring_dedent() { - check( + check_at( + crate::PythonVersion::PY314, indoc! {r#" a = "asdf" text = t""" diff --git a/crates/by_transforms/src/transforms/destructure.rs b/crates/by_transforms/src/transforms/destructure.rs index 9fe0e85b84..bdd9a0492b 100644 --- a/crates/by_transforms/src/transforms/destructure.rs +++ b/crates/by_transforms/src/transforms/destructure.rs @@ -58,8 +58,8 @@ use ruff_python_ast::visitor::{Visitor, walk_pattern, walk_stmt}; use ruff_python_ast::{ - AnyParameterRef, Expr, ModModule, Parameters, Pattern, PatternMatchAnd, PythonVersion, Stmt, - StmtFor, StmtLet, StmtMatch, StmtWith, + AnyParameterRef, Expr, ModModule, Parameters, Pattern, PatternMatchAnd, Stmt, StmtFor, StmtLet, + StmtMatch, StmtWith, }; use ruff_python_parser::semantic_errors::AND_PATTERN_IN_ALTERNATIVE; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; @@ -68,21 +68,13 @@ use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{AstPass, Fragment, PassContext}; use super::source_util::{line_indent, line_start, temporary_name}; -/// `match` statements — which every destructuring lowers to — are python 3.10 -/// syntax. -const MIN_VERSION: PythonVersion = PythonVersion::PY310; - pub(crate) struct DestructurePass<'src> { source: &'src str, - min_version: PythonVersion, } impl<'src> DestructurePass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { - Self { - source, - min_version, - } + pub(crate) fn new(source: &'src str) -> Self { + Self { source } } } @@ -92,7 +84,6 @@ impl AstPass for DestructurePass<'_> { source: self.source, edits: Vec::new(), errors: Vec::new(), - supported: self.min_version >= MIN_VERSION, names: NameGen::default(), }; for stmt in &module.body { @@ -107,7 +98,6 @@ struct DestructureLower<'src> { source: &'src str, edits: Vec<(TextRange, Vec)>, errors: Vec, - supported: bool, names: NameGen, } @@ -133,19 +123,6 @@ impl DestructureLower<'_> { 1 + self.source[..usize::from(offset)].matches('\n').count() } - /// Whether a destructuring at `offset` can be lowered at all, reporting the - /// python version it needs when it cannot. - fn supported_at(&mut self, offset: TextSize) -> bool { - if !self.supported { - self.errors.push(format!( - "destructuring needs python 3.10 or later (it lowers to a `match` statement) \ - (line {})", - self.line_of(offset), - )); - } - self.supported - } - /// The end of the `:` that ends the header of the block `body` belongs to. /// /// It is the last colon before the body starts: a return annotation or an @@ -215,7 +192,7 @@ impl DestructureLower<'_> { /// basedpython `let := [else: ...]`. fn lower_let(&mut self, let_stmt: &StmtLet) { - if !self.supported_at(let_stmt.range().start()) || !self.alone_on_its_line(let_stmt) { + if !self.alone_on_its_line(let_stmt) { return; } let indent = line_indent(self.source, let_stmt.range().start()).to_owned(); @@ -272,9 +249,6 @@ impl DestructureLower<'_> { let Some(pattern) = for_stmt.pattern.as_deref() else { return; }; - if !self.supported_at(for_stmt.range().start()) { - return; - } let Some(colon_end) = self.header_colon_end(for_stmt.iter.range().end(), &for_stmt.body) else { // an empty body cannot read the captures, so there is nothing to bind @@ -307,9 +281,6 @@ impl DestructureLower<'_> { if destructuring.is_empty() { return; } - if !self.supported_at(with_stmt.range().start()) { - return; - } let after_items = with_stmt .items @@ -341,9 +312,6 @@ impl DestructureLower<'_> { let Some((first, _)) = destructuring.first() else { return; }; - if !self.supported_at(first.range().start()) { - return; - } let Some(colon_end) = self.header_colon_end(parameters.range().end(), body) else { // an empty body cannot read the captures, so there is nothing to bind @@ -459,9 +427,6 @@ impl DestructureLower<'_> { { return; } - if !self.supported_at(match_stmt.range().start()) { - return; - } let indent = line_indent(self.source, match_stmt.range().start()).to_owned(); let case_indent = format!("{indent} "); @@ -1257,17 +1222,18 @@ mod tests { assert!(out.contains(" z = "), "got:\n{out}"); } + /// the `match` a destructuring lowers to is itself lowered for a target + /// that predates it, so the construct reaches every version the polyfill + /// covers rather than being an error below 3.10 #[test] - fn needs_python_310() { + fn lowers_below_python_310() { let config = Config { min_version: PythonVersion::PY39, ..Config::test_default() }; - let err = transpile("let (a, b) := (1, 2)\n", &config).unwrap_err(); - assert!( - err.contains("destructuring needs python 3.10"), - "got:\n{err}" - ); - assert!(err.contains("(line 1)"), "reports the line, got:\n{err}"); + let out = transpile("let (a, b) := (1, 2)\n", &config).unwrap(); + assert!(!out.contains("match "), "got:\n{out}"); + assert!(out.contains("a := "), "binds the captures, got:\n{out}"); + assert!(out.contains("b := "), "binds the captures, got:\n{out}"); } } diff --git a/crates/by_transforms/src/transforms/if_let.rs b/crates/by_transforms/src/transforms/if_let.rs index 848b4edc31..f7c6ea1494 100644 --- a/crates/by_transforms/src/transforms/if_let.rs +++ b/crates/by_transforms/src/transforms/if_let.rs @@ -35,7 +35,7 @@ //! namespace. use ruff_python_ast::visitor::{Visitor, walk_stmt}; -use ruff_python_ast::{Expr, Pattern, PythonVersion, Stmt, StmtIf}; +use ruff_python_ast::{Expr, Pattern, Stmt, StmtIf}; use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; use ruff_text_size::{Ranged, TextRange, TextSize}; @@ -44,20 +44,13 @@ use super::destructure::{NameGen, push_destructure}; use super::source_util::{line_indent, temporary_name}; use crate::type_info::TypeInfo; -/// `match` statements — which the lowering emits — are python 3.10 syntax. -const MIN_VERSION: PythonVersion = PythonVersion::PY310; - pub(crate) struct IfLetPass<'src> { source: &'src str, - min_version: PythonVersion, } impl<'src> IfLetPass<'src> { - pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { - Self { - source, - min_version, - } + pub(crate) fn new(source: &'src str) -> Self { + Self { source } } } @@ -69,7 +62,6 @@ impl TypeAwarePass for IfLetPass<'_> { edits: Vec::new(), errors: Vec::new(), counter: 0, - supported: self.min_version >= MIN_VERSION, names: NameGen::default(), }; for stmt in stmts { @@ -95,7 +87,6 @@ struct IfLetLower<'a, 'src> { errors: Vec, /// monotonic across the file so sibling chains get distinct selectors counter: usize, - supported: bool, /// names the temporaries a clause's destructuring needs names: NameGen, } @@ -156,15 +147,6 @@ impl IfLetLower<'_, '_> { return; } - if !self.supported { - self.errors.push(format!( - "`if let` needs python 3.10 or later (it lowers to a `match` statement) \ - (line {})", - self.line_of(if_stmt.range().start()), - )); - return; - } - let mut clauses = vec![Clause { start: if_stmt.range().start(), pattern: if_stmt.pattern.as_deref(), @@ -533,13 +515,15 @@ mod tests { ); } + /// the `match` this lowering emits is itself lowered for a target that + /// predates it, so an `if let` reaches every version the polyfill covers #[test] - fn needs_python_310() { + fn lowers_below_python_310() { let config = Config { min_version: PythonVersion::PY39, ..Config::test_default() }; - let err = transpile( + let out = transpile( indoc! {" opt: int | None = 1 if let int(x) := opt: @@ -547,30 +531,9 @@ mod tests { "}, &config, ) - .unwrap_err(); - assert!(err.contains("`if let` needs python 3.10"), "got:\n{err}"); - assert!(err.contains("(line 2)"), "reports the line, got:\n{err}"); - } - - /// a chain nested in another statement is reported too — the walk descends - /// past the statement it could not lower - #[test] - fn needs_python_310_when_nested() { - let config = Config { - min_version: PythonVersion::PY39, - ..Config::test_default() - }; - let err = transpile( - indoc! {" - def f(opt: int | None): - if opt: - if let int(x) := opt: - print(x) - "}, - &config, - ) - .unwrap_err(); - assert!(err.contains("`if let` needs python 3.10"), "got:\n{err}"); - assert!(err.contains("(line 3)"), "reports the line, got:\n{err}"); + .unwrap(); + assert!(!out.contains("match "), "got:\n{out}"); + assert!(out.contains("x := "), "binds the capture, got:\n{out}"); + assert!(out.contains("print(x)"), "keeps the body, got:\n{out}"); } } diff --git a/crates/by_transforms/src/transforms/kw_subscript.rs b/crates/by_transforms/src/transforms/kw_subscript.rs index 075111f094..49bd332ae8 100644 --- a/crates/by_transforms/src/transforms/kw_subscript.rs +++ b/crates/by_transforms/src/transforms/kw_subscript.rs @@ -8,7 +8,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt}; +use ruff_python_ast::{Expr, PythonVersion, Stmt}; use ruff_text_size::{Ranged, TextRange}; use crate::transforms::ast_driver::{PassContext, TypeAwarePass}; @@ -17,14 +17,20 @@ use crate::type_info::TypeInfo; pub(crate) struct KwSubscript<'src, T: TypeInfo + ?Sized> { source: &'src str, types: Option<&'src T>, + min_version: PythonVersion, pub(crate) edits: Vec, } impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { - pub(crate) fn new(source: &'src str, types: Option<&'src T>) -> Self { + pub(crate) fn new( + source: &'src str, + types: Option<&'src T>, + min_version: PythonVersion, + ) -> Self { Self { source, types, + min_version, edits: Vec::new(), } } @@ -39,7 +45,7 @@ impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { /// import for nested `??` is still raised by `OptionalTypePass`, which walks /// every expression independently fn value_src(&self, expr: &Expr) -> String { - crate::transforms::optional_type::rewrite_type_expr(self.source, expr) + crate::transforms::optional_type::rewrite_type_expr(self.source, expr, self.min_version) .unwrap_or_else(|| self.src(expr.range()).to_owned()) } @@ -307,17 +313,22 @@ impl<'src, T: TypeInfo + ?Sized> KwSubscript<'src, T> { pub(crate) struct KwSubscriptPass<'src> { source: &'src str, + min_version: PythonVersion, } impl<'src> KwSubscriptPass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + Self { + source, + min_version, + } } } impl TypeAwarePass for KwSubscriptPass<'_> { fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut inner: KwSubscript<'_, dyn TypeInfo> = KwSubscript::new(self.source, Some(types)); + let mut inner: KwSubscript<'_, dyn TypeInfo> = + KwSubscript::new(self.source, Some(types), self.min_version); for stmt in stmts { inner.visit_stmt(stmt); } diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs new file mode 100644 index 0000000000..1dca86dcbb --- /dev/null +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -0,0 +1,863 @@ +//! Lowering of the `match` statement for python versions that predate it. +//! +//! `match` is python 3.10 syntax, so a target below that cannot even parse a +//! file containing one. Every `match` in the emitted python — the ones the +//! author wrote, and the ones other lowerings produce (`let` destructuring, +//! `if let`, statement expressions, enum exhaustiveness) — is rewritten here +//! into an `if`/`elif` chain whose conditions do the matching: +//! +//! ```text +//! match point: if [__by_match_0__ := (point)]: +//! case Point(0, y): if isinstance(__by_match_0__, (__by_match_1__ := (Point))) and … +//! north(y) ⇒ north(y) +//! case _: else: +//! elsewhere() elsewhere() +//! ``` +//! +//! The rewrite replaces *only* the header spans — `match … :` and each +//! `case … :` — so every case body keeps its exact source bytes, at its exact +//! indentation. That is what makes the subject line become a wrapper `if` +//! rather than a plain assignment: the case clauses sit one level in from the +//! `match`, and they can only stay there if something opened a block on the +//! line the `match` occupied. `[name := subject]` is a one-element list, so it +//! is always truthy and the subject's own truthiness is never consulted. +//! +//! A header that spanned several lines is padded with blank lines to the same +//! count, so the statement occupies exactly the lines it did before and the +//! `.by` line map stays true. +//! +//! ## Patterns as expressions +//! +//! A case's pattern becomes one boolean expression over the subject, with +//! captures bound by assignment expressions along the way — which is why the +//! lowering itself needs python 3.8. Structure that python cannot ask for in an +//! expression is asked of the small helper functions in [`preamble`]: whether a +//! value counts as a sequence or a mapping, what a class's `__match_args__` +//! names, and how a missing attribute or key reports itself. Those helpers +//! answer "no match" with the `_by_match_miss` sentinel, never with an +//! exception, so a failed sub-pattern falls through to the next case instead of +//! escaping the statement. +//! +//! Sub-subjects are bound to temporaries as they are reached, so a nested +//! pattern reads its value once, in the order the source wrote it. +//! +//! ## What is not reproduced +//! +//! A temporary is left bound after the statement — python has no expression +//! that unbinds a name, and a `del` statement would cost a line the line map +//! cannot spare. The names are dunders (see +//! [`temporary_name`](super::source_util::temporary_name)) so that a `match` in +//! a class body leaves nothing `enum` or `dataclass` would read as a member. +//! +//! A comment written *inside* a multi-line pattern is dropped, since the +//! pattern's own text is replaced by the test compiled from it. Comments +//! anywhere else — including one after the header colon — are untouched. +//! +//! Nothing here reports a failure. A `match` this cannot lower — one aimed at a +//! python older than assignment expressions — is left standing, and the +//! target-version check that runs after this reports it against the `.by` line +//! the author actually wrote, which is a coordinate the lowering (working on +//! generated python) does not have. + +use ruff_python_ast::visitor::{Visitor, walk_stmt}; +use ruff_python_ast::{ + Pattern, PatternMatchClass, PatternMatchMapping, PatternMatchSequence, PySourceType, + PythonVersion, Singleton, Stmt, StmtMatch, +}; +use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use super::source_util::{preamble_offset, temporary_name}; + +/// the version that understands `match` natively; at or above it nothing here runs +const MATCH_VERSION: PythonVersion = PythonVersion::PY310; + +/// the version the lowering itself needs — it binds captures with assignment +/// expressions, which arrived in 3.8 +const LOWERING_VERSION: PythonVersion = PythonVersion::PY38; + +/// the sentinel a helper returns for "this sub-pattern did not match" +const MISS: &str = "_by_match_miss"; + +/// Rewrite every `match` statement in `source` for a target that predates them. +pub(crate) fn lower(source: String, min_version: PythonVersion) -> String { + if !(LOWERING_VERSION..MATCH_VERSION).contains(&min_version) { + return source; + } + + let parsed = ruff_python_parser::parse_unchecked_source(&source, PySourceType::Python); + let mut lower = Lower { + source: &source, + edits: Vec::new(), + counter: 0, + needs: Needs::default(), + }; + for stmt in parsed.suite() { + lower.visit_stmt(stmt); + } + + let (edits, needs) = (lower.edits, lower.needs); + if edits.is_empty() { + return source; + } + + // a file whose every pattern is a plain value test names no sentinel + let sentinel = edits + .iter() + .any(|(_, replacement)| replacement.contains(MISS)); + let body = apply(&source, edits); + let preamble = preamble(&needs, sentinel); + let at = preamble_offset(&body); + format!("{}{preamble}{}", &body[..at], &body[at..]) +} + +/// Apply disjoint replacements, ascending by start. +fn apply(source: &str, mut edits: Vec<(TextRange, String)>) -> String { + edits.sort_by_key(|(range, _)| range.start()); + let mut out = String::with_capacity(source.len()); + let mut at = 0usize; + for (range, replacement) in edits { + let start = usize::from(range.start()); + if start < at { + continue; + } + out.push_str(&source[at..start]); + out.push_str(&replacement); + at = usize::from(range.end()); + } + out.push_str(&source[at..]); + out +} + +/// Which runtime helpers the lowered file ends up naming. Only the ones a +/// pattern actually reached are emitted, so a file whose only `match` tests +/// literals carries no preamble beyond the sentinel. +#[derive(Default)] +#[expect( + clippy::struct_excessive_bools, + reason = "one independent flag per helper the preamble can emit" +)] +struct Needs { + sequence: bool, + mapping: bool, + mapping_rest: bool, + class_positional: bool, + class_keyword: bool, +} + +struct Lower<'src> { + source: &'src str, + edits: Vec<(TextRange, String)>, + /// monotonic across the file, so every temporary is distinct + counter: usize, + needs: Needs, +} + +impl<'ast> Visitor<'ast> for Lower<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + if let Stmt::Match(match_stmt) = stmt { + self.lower_match(match_stmt); + } + walk_stmt(self, stmt); + } +} + +impl Lower<'_> { + /// A temporary name nothing else in the file spells. + fn fresh(&mut self) -> String { + loop { + let name = temporary_name("match", self.counter); + self.counter += 1; + if !self.source.contains(&name) { + return name; + } + } + } + + fn src(&self, range: TextRange) -> &str { + &self.source[usize::from(range.start())..usize::from(range.end())] + } + + /// A span of the original source as an expression: always parenthesized, so + /// that a fragment carrying its own line breaks or a trailing comment stays + /// inside a bracketed continuation instead of ending the generated line. + fn expr_src(&self, range: TextRange) -> String { + format!("({})", self.src(range)) + } + + /// The end of the header colon that follows `after` — the end of a subject, + /// a pattern, or a guard. Only closing brackets, commas, whitespace and + /// comments can stand between the two. + fn colon_end(&self, after: TextSize) -> Option { + SimpleTokenizer::starts_at(after, self.source) + .skip_trivia() + .find(|token| { + !matches!( + token.kind(), + SimpleTokenKind::RParen + | SimpleTokenKind::RBracket + | SimpleTokenKind::RBrace + | SimpleTokenKind::Comma + ) + }) + .filter(|token| token.kind() == SimpleTokenKind::Colon) + .map(|token| token.range().end()) + } + + fn lower_match(&mut self, match_stmt: &StmtMatch) { + let subject = match_stmt.subject.range(); + let Some(header_colon) = self.colon_end(subject.end()) else { + return; + }; + + let subject_name = self.fresh(); + let mut headers = Vec::with_capacity(match_stmt.cases.len()); + for (index, case) in match_stmt.cases.iter().enumerate() { + let test_end = case + .guard + .as_ref() + .map_or_else(|| case.pattern.range().end(), |guard| guard.range().end()); + let Some(colon) = self.colon_end(test_end) else { + return; + }; + + let mut test = self.compile(&case.pattern, &subject_name); + if let Some(guard) = &case.guard { + test = conjoin(vec![test, self.expr_src(guard.range())]); + } + + let span = TextRange::new(case.range().start(), colon); + let last = index + 1 == match_stmt.cases.len(); + let replacement = if index == 0 { + format!("if {test}:") + } else if last && test == "True" { + "else:".to_owned() + } else { + format!("elif {test}:") + }; + headers.push((span, replacement)); + } + + self.edits.push(( + TextRange::new(match_stmt.range().start(), subject.start()), + format!("if [{subject_name} := ("), + )); + self.edits.push(( + TextRange::new(subject.end(), header_colon), + ")]:".to_owned(), + )); + for (span, replacement) in headers { + let padded = self.pad_to_span(span, replacement); + self.edits.push((span, padded)); + } + } + + /// Keep the replacement on as many lines as the header it replaces, by + /// appending the blank lines it is short. A blank line between a header and + /// its body is ignored by python, so the padding costs nothing but keeps + /// every later line where the line map says it is. + fn pad_to_span(&self, span: TextRange, replacement: String) -> String { + let original = self.src(span).matches('\n').count(); + let produced = replacement.matches('\n').count(); + let mut padded = replacement; + for _ in 0..original.saturating_sub(produced) { + padded.push('\n'); + } + padded + } + + /// The test that decides whether `pattern` matches the value already bound + /// to the name `subject`, binding the pattern's captures as it goes. + fn compile(&mut self, pattern: &Pattern, subject: &str) -> String { + match pattern { + Pattern::MatchAs(as_pattern) => { + let mut parts = Vec::new(); + if let Some(inner) = &as_pattern.pattern { + parts.push(self.compile(inner, subject)); + } + if let Some(name) = &as_pattern.name { + parts.push(format!("({} := {subject}) is not {MISS}", name.id)); + } + conjoin(parts) + } + Pattern::MatchSingleton(singleton) => { + let value = match singleton.value { + Singleton::None => "None", + Singleton::True => "True", + Singleton::False => "False", + }; + format!("{subject} is {value}") + } + Pattern::MatchValue(value) => { + format!("{subject} == {}", self.expr_src(value.value.range())) + } + Pattern::MatchOr(or_pattern) => { + let mut alternatives = Vec::with_capacity(or_pattern.patterns.len()); + for alternative in &or_pattern.patterns { + alternatives.push(self.compile(alternative, subject)); + } + disjoin(&alternatives) + } + Pattern::MatchAnd(and_pattern) => { + let mut conjuncts = Vec::with_capacity(and_pattern.patterns.len()); + for conjunct in &and_pattern.patterns { + conjuncts.push(self.compile(conjunct, subject)); + } + conjoin(conjuncts) + } + Pattern::MatchSequence(sequence) => self.compile_sequence(sequence, subject), + Pattern::MatchMapping(mapping) => self.compile_mapping(mapping, subject), + Pattern::MatchClass(class) => self.compile_class(class, subject), + // a star only appears as an element of a sequence pattern, where + // the sequence itself binds it + Pattern::MatchStar(_) => "True".to_owned(), + } + } + + /// Read `access` into a fresh temporary and match `pattern` against it. The + /// sentinel test is what turns a helper's "no match" answer into a failed + /// sub-pattern; for an access that cannot miss (a sequence element) it is + /// simply always true, and the binding is the point. + fn bind(&mut self, access: &str, pattern: &Pattern) -> String { + let temporary = self.fresh(); + let inner = self.compile(pattern, &temporary); + let read = format!("({temporary} := {access}) is not {MISS}"); + if inner == "True" { + read + } else { + conjoin(vec![read, inner]) + } + } + + /// A sequence pattern reads only the elements it has something to say + /// about, and reads the ones at fixed positions before it materializes the + /// list a `*rest` captures — which is the order python itself uses, and the + /// reason `[1, *rest, 9]` can be tried against a `deque` (whose elements + /// are indexable but whose slices are not) without raising. + fn compile_sequence(&mut self, sequence: &PatternMatchSequence, subject: &str) -> String { + self.needs.sequence = true; + let mut parts = vec![format!("_by_match_seq({subject})")]; + let star = sequence.patterns.iter().position(Pattern::is_match_star); + let fixed = match star { + None => { + parts.push(format!("len({subject}) == {}", sequence.patterns.len())); + sequence.patterns.len() + } + Some(star) => { + parts.push(format!("len({subject}) >= {}", sequence.patterns.len() - 1)); + star + } + }; + for (index, element) in sequence.patterns[..fixed].iter().enumerate() { + if is_wildcard(element) { + continue; + } + parts.push(self.bind(&format!("{subject}[{index}]"), element)); + } + if let Some(star) = star { + // elements after the star are counted from the end, so however much + // the star swallowed never enters into their index + let after = sequence.patterns.len() - star - 1; + for (offset, element) in sequence.patterns[star + 1..].iter().enumerate() { + if is_wildcard(element) { + continue; + } + let index = offset.cast_signed() - after.cast_signed(); + parts.push(self.bind(&format!("{subject}[{index}]"), element)); + } + if let Pattern::MatchStar(rest) = &sequence.patterns[star] + && let Some(name) = &rest.name + { + let slice = if after == 0 { + format!("{subject}[{star}:]") + } else { + format!("{subject}[{star}:-{after}]") + }; + parts.push(format!("({} := list({slice})) is not {MISS}", name.id)); + } + } + conjoin(parts) + } + + fn compile_mapping(&mut self, mapping: &PatternMatchMapping, subject: &str) -> String { + self.needs.mapping = true; + let mut parts = vec![format!("_by_match_map({subject})")]; + // each key is read into a temporary so it is evaluated exactly once, + // however many times the lowering goes on to name it + let mut keys = Vec::with_capacity(mapping.keys.len()); + for (key, value) in mapping.keys.iter().zip(&mapping.patterns) { + let name = self.fresh(); + parts.push(format!( + "({name} := {}) is not {MISS}", + self.expr_src(key.range()) + )); + parts.push(self.bind(&format!("_by_match_key({subject}, {name})"), value)); + keys.push(name); + } + if let Some(rest) = &mapping.rest { + self.needs.mapping_rest = true; + let matched = keys.iter().fold(String::new(), |mut matched, key| { + matched.push_str(key); + matched.push_str(", "); + matched + }); + parts.push(format!( + "({} := _by_match_rest({subject}, ({matched}))) is not {MISS}", + rest.id + )); + } + conjoin(parts) + } + + fn compile_class(&mut self, class: &PatternMatchClass, subject: &str) -> String { + let class_name = self.fresh(); + let mut parts = vec![format!( + "isinstance({subject}, ({class_name} := {}))", + self.expr_src(class.cls.range()) + )]; + + let positional = &class.arguments.patterns; + if !positional.is_empty() { + self.needs.class_positional = true; + let args = self.fresh(); + parts.push(format!( + "({args} := _by_match_args({class_name}, {subject}, {})) is not {MISS}", + positional.len() + )); + for (index, element) in positional.iter().enumerate() { + parts.push(self.bind(&format!("{args}[{index}]"), element)); + } + } + for keyword in &class.arguments.keywords { + self.needs.class_keyword = true; + parts.push(self.bind( + &format!("_by_match_attr({subject}, \"{}\")", keyword.attr.id), + &keyword.pattern, + )); + } + conjoin(parts) + } +} + +/// `_`: matches anything and binds nothing, so a sequence element it stands for +/// is never read. +fn is_wildcard(pattern: &Pattern) -> bool { + matches!( + pattern, + Pattern::MatchAs(as_pattern) + if as_pattern.pattern.is_none() && as_pattern.name.is_none() + ) +} + +/// Wrap `test` where it would otherwise be re-associated by a surrounding +/// operator. The check errs towards wrapping: an extra pair of parentheses is +/// always sound, a missing one is not. +fn paren(test: &str) -> String { + if test.contains(" and ") || test.contains(" or ") { + format!("({test})") + } else { + test.to_owned() + } +} + +fn conjoin(parts: Vec) -> String { + let parts: Vec = parts + .into_iter() + .filter(|part| part != "True") + .map(|part| paren(&part)) + .collect(); + if parts.is_empty() { + "True".to_owned() + } else { + parts.join(" and ") + } +} + +/// Alternatives are never dropped, however certain one of them looks: an +/// alternative that always matches still leaves the ones before it deciding +/// which captures get bound. +fn disjoin(parts: &[String]) -> String { + if parts.is_empty() { + return "True".to_owned(); + } + parts + .iter() + .map(|part| paren(part)) + .collect::>() + .join(" or ") +} + +/// The runtime the lowered tests call into. Every helper answers "no match" +/// with the sentinel rather than by raising, so that a subject which simply +/// lacks an attribute or a key falls through to the next case — while a subject +/// whose class is malformed (a `__match_args__` that is not a tuple of names) +/// still raises the `TypeError` python raises for it. +fn preamble(needs: &Needs, sentinel: bool) -> String { + let mut out = String::new(); + if sentinel { + out.push_str("_by_match_miss = object()\n"); + } + + if needs.sequence || needs.mapping { + out.push_str("import collections.abc as _by_match_abc\n"); + } + if needs.sequence { + // python decides "is a sequence" by a type flag rather than by an ABC, + // and sets it on a handful of builtins that register no ABC of their + // own. str, bytes and bytearray carry the flag's opposite: they are + // sequences everywhere else, and never match a sequence pattern + out.push_str("import array as _by_match_array\n"); + out.push_str( + "_by_match_seq_types = (list, tuple, range, memoryview, _by_match_array.array, \ + _by_match_abc.Sequence)\n", + ); + out.push_str("def _by_match_seq(subject):\n"); + out.push_str( + " return isinstance(subject, _by_match_seq_types) and not isinstance(subject, \ + (str, bytes, bytearray))\n", + ); + } + if needs.mapping { + out.push_str("def _by_match_map(subject):\n"); + out.push_str(" return isinstance(subject, _by_match_abc.Mapping)\n"); + out.push_str("def _by_match_key(subject, key):\n"); + out.push_str(" try:\n"); + out.push_str(" return subject[key]\n"); + out.push_str(" except KeyError:\n"); + out.push_str(" return _by_match_miss\n"); + } + if needs.mapping_rest { + out.push_str("def _by_match_rest(subject, matched):\n"); + out.push_str( + " return {key: value for key, value in subject.items() if key not in matched}\n", + ); + } + if needs.class_positional { + // a handful of builtins take one positional sub-pattern that matches + // the subject itself, in place of reading `__match_args__` + out.push_str( + "_by_match_self = (bool, bytearray, bytes, dict, float, frozenset, int, list, set, \ + str, tuple)\n", + ); + out.push_str("def _by_match_args(cls, subject, count):\n"); + out.push_str(" if cls in _by_match_self:\n"); + out.push_str(" if count > 1:\n"); + out.push_str( + " raise TypeError(f\"{cls.__name__}() accepts 1 positional sub-pattern \ + ({count} given)\")\n", + ); + out.push_str(" return (subject,)\n"); + out.push_str(" args = getattr(cls, \"__match_args__\", ())\n"); + out.push_str(" if not isinstance(args, tuple):\n"); + out.push_str( + " raise TypeError(f\"{cls.__name__}.__match_args__ must be a tuple \ + (got {type(args).__name__})\")\n", + ); + out.push_str(" if count > len(args):\n"); + out.push_str( + " raise TypeError(f\"{cls.__name__}() accepts {len(args)} positional \ + sub-patterns ({count} given)\")\n", + ); + out.push_str(" values = []\n"); + out.push_str(" for name in args[:count]:\n"); + out.push_str(" if not isinstance(name, str):\n"); + out.push_str( + " raise TypeError(f\"__match_args__ elements must be strings \ + (got {type(name).__name__})\")\n", + ); + out.push_str(" try:\n"); + out.push_str(" values.append(getattr(subject, name))\n"); + out.push_str(" except AttributeError:\n"); + out.push_str(" return _by_match_miss\n"); + out.push_str(" return tuple(values)\n"); + } + if needs.class_keyword { + out.push_str("def _by_match_attr(subject, name):\n"); + out.push_str(" try:\n"); + out.push_str(" return getattr(subject, name)\n"); + out.push_str(" except AttributeError:\n"); + out.push_str(" return _by_match_miss\n"); + } + out +} + +#[cfg(test)] +mod tests { + use crate::{Config, PythonVersion, transpile}; + use indoc::indoc; + + /// transpile for a target that predates `match` + fn check(input: &str, expected: &str) { + let config = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + assert_eq!(transpile(input, &config).unwrap(), expected); + } + + fn lowered(input: &str) -> String { + let config = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + transpile(input, &config).unwrap() + } + + /// the subject line opens the block the case clauses already sit inside, so + /// nothing is re-indented and the value's own truthiness is never consulted + #[test] + fn value_patterns() { + check( + indoc! {r#" + def f(n: int) -> str: + match n: + case 0: + return "zero" + case 1 | 2: + return "small" + case _: + return "many" + "#}, + indoc! {r#" + from __future__ import annotations + def f(n: int) -> str: + if [__by_match_0__ := (n)]: + if __by_match_0__ == (0): + return "zero" + elif __by_match_0__ == (1) or __by_match_0__ == (2): + return "small" + else: + return "many" + "#}, + ); + } + + /// `None`, `True` and `False` are matched by identity, not equality + #[test] + fn singletons() { + let out = lowered( + "def f(v: object):\n match v:\n case None:\n pass\n case True:\n pass\n", + ); + assert!(out.contains("is None"), "got:\n{out}"); + assert!(out.contains("is True"), "got:\n{out}"); + } + + /// a capture always matches, so the sentinel comparison beside it is only + /// there to make the binding an expression + #[test] + fn captures_bind_in_the_enclosing_scope() { + check( + indoc! {" + def f(v: object): + match v: + case got: + print(got) + "}, + indoc! {" + from __future__ import annotations + _by_match_miss = object() + def f(v: object): + if [__by_match_0__ := (v)]: + if (got := __by_match_0__) is not _by_match_miss: + print(got) + "}, + ); + } + + /// nothing names the sentinel when every pattern is a plain value test + #[test] + fn a_file_that_needs_no_runtime_carries_none() { + let out = lowered("def f(n: int):\n match n:\n case 0:\n pass\n"); + assert!(!out.contains("_by_match_miss"), "got:\n{out}"); + assert!(!out.contains("def _by_match"), "got:\n{out}"); + assert!(!out.contains("_by_match_abc"), "got:\n{out}"); + } + + /// a guard runs after the pattern bound its captures, and only then + #[test] + fn guards_follow_the_pattern() { + let out = lowered( + "def f(v: object):\n match v:\n case [a, b] if a < b:\n pass\n", + ); + assert!(out.contains("and (a < b)"), "got:\n{out}"); + } + + #[test] + fn class_patterns_read_match_args() { + let out = lowered(indoc! {" + class Point: + __match_args__ = ('x', 'y') + + def f(v: object): + match v: + case Point(x, y=0): + print(x) + "}); + assert!(out.contains("def _by_match_args("), "got:\n{out}"); + assert!(out.contains("def _by_match_attr("), "got:\n{out}"); + assert!( + out.contains("_by_match_args(__by_match_1__, __by_match_0__, 1)"), + "one positional sub-pattern, got:\n{out}" + ); + assert!( + out.contains("_by_match_attr(__by_match_0__, \"y\")"), + "got:\n{out}" + ); + } + + /// a sequence's fixed elements are read before the star's list is built, + /// and elements after the star are indexed from the end + #[test] + fn sequence_patterns() { + let out = lowered( + "def f(v: object):\n match v:\n case [1, *rest, last]:\n print(rest, last)\n", + ); + let fixed = out + .find("__by_match_0__[-1]") + .expect("reads the last element"); + let star = out.find("list(").expect("captures the rest"); + assert!(fixed < star, "fixed elements come first, got:\n{out}"); + assert!(out.contains("__by_match_0__[1:-1]"), "got:\n{out}"); + assert!(out.contains("len(__by_match_0__) >= 2"), "got:\n{out}"); + } + + /// an element the pattern says nothing about is never read, which is what + /// lets `[_, x]` be tried against a sequence whose first element raises + #[test] + fn a_wildcard_element_is_not_read() { + let out = lowered( + "def f(v: object):\n match v:\n case [_, second]:\n print(second)\n", + ); + assert!(!out.contains("__by_match_0__[0]"), "got:\n{out}"); + assert!(out.contains("__by_match_0__[1]"), "got:\n{out}"); + } + + /// each key is evaluated once, however many times the lowering names it + #[test] + fn mapping_patterns() { + let out = lowered(indoc! {r#" + def f(v: object): + match v: + case {"a": a, **rest}: + print(a, rest) + "#}); + assert!(out.contains("def _by_match_key("), "got:\n{out}"); + assert!(out.contains("def _by_match_rest("), "got:\n{out}"); + assert!( + out.contains("(__by_match_1__ := (\"a\"))"), + "the key is read into a temporary, got:\n{out}" + ); + assert!( + out.contains("_by_match_rest(__by_match_0__, (__by_match_1__, ))"), + "and the rest is the mapping minus that same temporary, got:\n{out}" + ); + } + + /// a header spread over several lines is replaced by one line plus the + /// blank lines it is short, so everything below it keeps its line number + #[test] + fn a_multiline_header_keeps_its_height() { + let input = indoc! {" + def f(v: object): + match ( + v, + ): + case [ + a, + ]: + return a + case _: + return None + "}; + let out = lowered(input); + let generated = out + .lines() + .take_while(|line| *line != "def f(v: object):") + .count(); + assert_eq!( + out.lines().count() - generated, + input.lines().count(), + "got:\n{out}" + ); + } + + /// a nested `match` lowers on its own terms; only headers are replaced, so + /// the outer statement's bodies are untouched either way + #[test] + fn nested_matches() { + let out = lowered(indoc! {" + def f(v: object): + match v: + case [head, *_]: + match head: + case 0: + return 'zero' + case _: + return None + "}); + assert!(!out.contains("match "), "got:\n{out}"); + assert!(out.contains("__by_match_2__ := (head)"), "got:\n{out}"); + } + + /// a target that has `match` keeps it + #[test] + fn untouched_from_python_310() { + let out = transpile( + "def f(n: int):\n match n:\n case 0:\n pass\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("match n:"), "got:\n{out}"); + assert!(!out.contains("_by_match_miss"), "got:\n{out}"); + assert!(!out.contains("def _by_match"), "got:\n{out}"); + assert!(!out.contains("_by_match_abc"), "got:\n{out}"); + } + + /// the lowering binds with assignment expressions, so a target older than + /// those is left to the target-version check rather than lowered into + /// something that version cannot run either + #[test] + fn python_37_declines_and_is_reported() { + let config = Config { + min_version: PythonVersion::PY37, + ..Config::test_default() + }; + let err = transpile( + "def f(n: int):\n match n:\n case 0:\n pass\n", + &config, + ) + .unwrap_err(); + assert!( + err.contains("Cannot use `match` statement on Python 3.7"), + "got:\n{err}" + ); + } + + /// the temporaries a `match` in a class body leaves behind are dunders, so + /// `enum` and `dataclass` read them as machinery rather than as members + #[test] + fn class_body_leftovers_are_dunders() { + let out = lowered(indoc! {" + class A: + match 'x': + case str() as which: + label = which + "}); + for line in out.lines() { + let assigned = line.trim_start().split(" :=").next().unwrap_or_default(); + if assigned.starts_with("_by_match") { + continue; + } + assert!( + !line.contains("__by_match") + || line.contains("__by_match_0__ := ") + || line.contains("__by_match"), + "got:\n{out}" + ); + } + assert!( + out.matches("__by_match").count() > 0 && !out.contains("_by_match_0 "), + "got:\n{out}" + ); + } +} diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index ecdf594fac..13b4f82b1a 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -43,6 +43,7 @@ pub(crate) mod literal_string; pub(crate) mod literal_types; pub(crate) mod local_once; pub(crate) mod main_function; +pub(crate) mod match_polyfill; pub(crate) mod match_type; pub(crate) mod modifiers; pub(crate) mod mutable_defaults; @@ -58,6 +59,7 @@ pub(crate) mod protocol_type; pub(crate) mod raises_clause; pub(crate) mod reified_generic; pub(crate) mod repeated_underscore; +pub(crate) mod runtime_union; pub(crate) mod sentinel; pub(crate) mod some_ctor; pub(crate) mod soundness; diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index fdd3ad005f..0c163053a1 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -264,8 +264,16 @@ mod tests { use indoc::indoc; fn check(input: &str, expected: &str) { + check_at(crate::Config::test_default().min_version, input, expected); + } + + fn check_at(min_version: crate::PythonVersion, input: &str, expected: &str) { + let config = crate::Config { + min_version, + ..crate::Config::test_default() + }; assert_eq!( - transpile(input, &crate::Config::test_default()).unwrap(), + transpile(input, &config).unwrap(), crate::python_passthrough::lazify_expected(expected) ); } @@ -616,9 +624,11 @@ mod tests { ); } + /// t-strings are 3.14 syntax, so the target has to be one that can run them #[test] fn tstring_default() { - check( + check_at( + crate::PythonVersion::PY314, indoc! {r#" data = "fdsa" def f(a=t"asdf{data}"): diff --git a/crates/by_transforms/src/transforms/optional_type.rs b/crates/by_transforms/src/transforms/optional_type.rs index fbed3543db..52f283476b 100644 --- a/crates/by_transforms/src/transforms/optional_type.rs +++ b/crates/by_transforms/src/transforms/optional_type.rs @@ -4,6 +4,11 @@ //! This pass rewrites it to the runtime-compatible union `T | None` //! (`Optional.Some(x)` is `x`, `Optional.None_` is `None`). //! +//! A target older than 3.10 has no `type.__or__`, and an optional is written in +//! plenty of places the runtime evaluates — a `cast` target, an alias — so for +//! those the union is spelled `Union[T, None]` instead. Which spelling is used +//! never changes what the type means. +//! //! It emits narrow text edits rather than mutating the AST so it composes with //! the value-position operator lowerings that share a statement — e.g. a //! function whose signature has `int?` and whose body uses `??` or `?.`. A @@ -20,7 +25,7 @@ //! runtime representation is still being settled. use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt, UnaryOp}; +use ruff_python_ast::{Expr, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{PassContext, TypeAwarePass}; @@ -35,6 +40,10 @@ struct OptionalLower<'src> { edits: Vec<(TextRange, String)>, /// set when any lowered optional produced a runtime `Optional[...]` wrapper needs_runtime: bool, + /// set when any lowered optional was spelled `Union[...]`, which needs the import + needs_union: bool, + /// whether the target can spell a union with `|` (python 3.10) + native_union: bool, source: &'src str, /// stack of in-scope PEP 695 type-parameter names. `?` over a bare type /// variable lowers to the *wrapped* form (`Optional[T | None]`) — a plain @@ -44,10 +53,12 @@ struct OptionalLower<'src> { } impl<'src> OptionalLower<'src> { - fn new(source: &'src str) -> Self { + fn new(source: &'src str, min_version: PythonVersion) -> Self { Self { edits: Vec::new(), needs_runtime: false, + needs_union: false, + native_union: min_version >= PythonVersion::PY310, source, typevar_scopes: Vec::new(), } @@ -120,13 +131,21 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { let wrap_layers = (depth - 1) as usize + usize::from(generic_operand); if wrap_layers >= 1 { self.needs_runtime = true; - self.edits.push(( - TextRange::empty(node.start()), - "Optional[".repeat(wrap_layers), - )); + } + let mut prefix = "Optional[".repeat(wrap_layers); + if !self.native_union { + self.needs_union = true; + prefix.push_str("Union["); + } + if !prefix.is_empty() { + self.edits.push((TextRange::empty(node.start()), prefix)); } let mut replacement = close_parens; - replacement.push_str(" | None"); + if self.native_union { + replacement.push_str(" | None"); + } else { + replacement.push_str(", None]"); + } for _ in 0..wrap_layers { replacement.push(']'); } @@ -141,11 +160,15 @@ impl<'ast> Visitor<'ast> for OptionalLower<'_> { pub(crate) struct OptionalTypePass<'src> { source: &'src str, + min_version: PythonVersion, } impl<'src> OptionalTypePass<'src> { - pub(crate) fn new(source: &'src str) -> Self { - Self { source } + pub(crate) fn new(source: &'src str, min_version: PythonVersion) -> Self { + Self { + source, + min_version, + } } } @@ -156,8 +179,12 @@ impl<'src> OptionalTypePass<'src> { /// lowered when that constructor renders its nested types. The runtime /// `Optional[...]` import for nested `T??` is handled by [`OptionalTypePass`], /// which independently walks every type position. -pub(crate) fn collect_edits(source: &str, expr: &Expr) -> Vec<(TextRange, String)> { - let mut lower = OptionalLower::new(source); +pub(crate) fn collect_edits( + source: &str, + expr: &Expr, + min_version: PythonVersion, +) -> Vec<(TextRange, String)> { + let mut lower = OptionalLower::new(source, min_version); lower.visit_expr(expr); lower.edits } @@ -165,8 +192,12 @@ pub(crate) fn collect_edits(source: &str, expr: &Expr) -> Vec<(TextRange, String /// Lower the optionals in a single type-expression subtree to a string, or /// `None` if it contains no optional. Used by type constructors (tuple type, /// kw-subscript) to lower a nested `T?` when they splice their element types. -pub(crate) fn rewrite_type_expr(source: &str, expr: &Expr) -> Option { - let mut edits = collect_edits(source, expr); +pub(crate) fn rewrite_type_expr( + source: &str, + expr: &Expr, + min_version: PythonVersion, +) -> Option { + let mut edits = collect_edits(source, expr, min_version); if edits.is_empty() { return None; } @@ -188,13 +219,17 @@ pub(crate) fn rewrite_type_expr(source: &str, expr: &Expr) -> Option { impl TypeAwarePass for OptionalTypePass<'_> { fn run(&self, stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { - let mut lower = OptionalLower::new(self.source); + let mut lower = OptionalLower::new(self.source, self.min_version); for stmt in stmts { lower.visit_stmt(stmt); } if lower.needs_runtime { ctx.required_imports.push(OPTIONAL_RUNTIME.to_owned()); } + if lower.needs_union { + ctx.required_imports + .push("from typing import Union".to_owned()); + } ctx.text_edits.extend(lower.edits); } } @@ -267,20 +302,40 @@ class Optional: ); } + /// below 3.10 there is no `type.__or__`, so the optional is spelled as the + /// `Union` that works on every version — and the annotation is deferred too, + /// which is what the future import is for #[test] - fn py39_target_defers_annotation_evaluation() { - // below 3.10 the runtime cannot evaluate the pep 604 union this very - // lowering produces, so the future import is mandatory + fn py39_target_spells_the_union_out() { let config = crate::Config { min_version: crate::PythonVersion::PY39, ..crate::Config::test_default() }; assert_eq!( crate::transpile("x: int? = None\n", &config).unwrap(), - "from __future__ import annotations\nx: int | None = None\n" + indoc! {" + from __future__ import annotations + from typing import Union + x: Union[int, None] = None + "} ); } + /// the spelling reaches a position the runtime really does evaluate + #[test] + fn py39_target_spells_a_cast_target_out() { + let config = crate::Config { + min_version: crate::PythonVersion::PY39, + ..crate::Config::test_default() + }; + let out = crate::transpile( + "from typing import cast\ndef f(v: object):\n return cast(int?, v)\n", + &config, + ) + .unwrap(); + assert!(out.contains("cast(Union[int, None], v)"), "got:\n{out}"); + } + #[test] fn generic_typevar_optional_wraps() { // `?` over a bare in-scope type variable is the wrapped form — a plain diff --git a/crates/by_transforms/src/transforms/runtime_union.rs b/crates/by_transforms/src/transforms/runtime_union.rs new file mode 100644 index 0000000000..a2e052cde1 --- /dev/null +++ b/crates/by_transforms/src/transforms/runtime_union.rs @@ -0,0 +1,343 @@ +//! Lowering of PEP 604 unions that reach the runtime, for targets before 3.10. +//! +//! `int | str` is a call of `type.__or__`, which python only grew in 3.10. +//! Written as an *annotation* that costs nothing — a target this old always +//! gets `from __future__ import annotations`, so no annotation is ever +//! evaluated — but written where the value is really produced it is a +//! `TypeError` at import time: +//! +//! ```text +//! isinstance(x, int | str) ⇒ isinstance(x, (int, str,)) +//! cast(int | str, value) ⇒ cast(Union[int, str], value) +//! ``` +//! +//! The two spellings are not interchangeable: `isinstance` takes a tuple of +//! classes and rejects a `typing.Union`, while everything else wants the +//! `Union` — so the classinfo argument of `isinstance` / `issubclass` is +//! rewritten to a tuple, and every other union to `Union[...]`. Within that +//! argument the tuple form reaches through tuples and lists, since `isinstance` +//! accepts those nested; anywhere else inside it — a subscript's slice, a call's +//! arguments — is ordinary value context again. +//! +//! An arm written as `None` becomes `type(None)` in the tuple form. `None` is a +//! value, not a class, and only the union operator accepts it as shorthand for +//! `NoneType`. +//! +//! Whether a `|` is a union at all is asked of the checker rather than guessed +//! from the shape: `a | b` is overwhelmingly a bitwise or, and only the types of +//! its operands tell the two apart. + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Expr, ExprCall, Operator, PythonVersion, Stmt}; +use ruff_text_size::{Ranged, TextRange}; + +use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; +use crate::type_info::TypeInfo; + +/// the version `type.__or__` arrived in +const MIN_VERSION: PythonVersion = PythonVersion::PY310; + +pub(crate) struct RuntimeUnionPass { + min_version: PythonVersion, +} + +impl RuntimeUnionPass { + pub(crate) fn new(min_version: PythonVersion) -> Self { + Self { min_version } + } +} + +impl TypeAwarePass for RuntimeUnionPass { + fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { + if self.min_version >= MIN_VERSION { + return; + } + let mut lower = Lower { + types, + edits: Vec::new(), + needs_import: false, + }; + for stmt in stmts { + lower.visit_stmt(stmt); + } + if lower.needs_import { + ctx.required_imports + .push("from typing import Union".to_owned()); + } + ctx.template_edits.extend(lower.edits); + } +} + +struct Lower<'a> { + types: &'a dyn TypeInfo, + edits: Vec<(TextRange, Vec)>, + needs_import: bool, +} + +impl<'ast> Visitor<'ast> for Lower<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + walk_stmt(self, stmt); + } + + /// an annotation is a string at runtime for every target this pass runs + /// for, so nothing in one is ever evaluated + fn visit_annotation(&mut self, _expr: &'ast Expr) {} + + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Call(call) = expr + && let Some(classinfo) = self.classinfo_argument(call) + { + for (index, argument) in call.arguments.args.iter().enumerate() { + if index == 1 { + self.visit_classinfo(classinfo); + } else { + self.visit_expr(argument); + } + } + for keyword in &call.arguments.keywords { + self.visit_expr(&keyword.value); + } + self.visit_expr(&call.func); + return; + } + + if let Some(arms) = self.union_arms(expr) { + self.needs_import = true; + let fragments = spell(&arms, Form::Union); + self.edits.push((expr.range(), fragments)); + for arm in arms { + self.visit_expr(arm); + } + return; + } + + walk_expr(self, expr); + } +} + +/// How a union is spelled, which is decided by where it stands. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Form { + /// a tuple of classes, for `isinstance` / `issubclass` + ClassInfo, + /// `Union[...]`, for everywhere else + Union, +} + +impl<'ast> Lower<'_> { + /// The second positional argument of a call to the real `isinstance` or + /// `issubclass`. A file that binds either name means something else by it, + /// and gets no special treatment. + fn classinfo_argument(&self, call: &'ast ExprCall) -> Option<&'ast Expr> { + let Expr::Name(name) = call.func.as_ref() else { + return None; + }; + if !matches!(name.id.as_str(), "isinstance" | "issubclass") { + return None; + } + if !self.types.is_unbound_at(name.id.as_str(), &call.func) { + return None; + } + call.arguments.args.get(1) + } + + /// The arms of `expr`, when it is a union the runtime will evaluate. + fn union_arms(&self, expr: &'ast Expr) -> Option> { + let Expr::BinOp(binop) = expr else { + return None; + }; + if binop.op != Operator::BitOr || !self.types.is_runtime_union(expr) { + return None; + } + let mut arms = Vec::new(); + collect_arms(expr, &mut arms); + Some(arms) + } + + /// Visit an expression standing where `isinstance` expects classes. A union + /// here becomes a tuple, and so does one nested inside a tuple or list the + /// argument already spells; everything else is ordinary value context. + fn visit_classinfo(&mut self, expr: &'ast Expr) { + if let Some(arms) = self.union_arms(expr) { + let fragments = spell(&arms, Form::ClassInfo); + self.edits.push((expr.range(), fragments)); + for arm in arms { + self.visit_classinfo(arm); + } + return; + } + match expr { + Expr::Tuple(tuple) => { + for element in &tuple.elts { + self.visit_classinfo(element); + } + } + Expr::List(list) => { + for element in &list.elts { + self.visit_classinfo(element); + } + } + _ => self.visit_expr(expr), + } + } +} + +/// The union as the target can spell it. Each arm passes through as source so +/// that a lowering inside it — an optional `T?`, a tuple type — still composes. +fn spell(arms: &[&Expr], form: Form) -> Vec { + let mut fragments = Vec::with_capacity(arms.len() * 2 + 2); + fragments.push(Fragment::Lit( + match form { + Form::ClassInfo => "(", + Form::Union => "Union[", + } + .to_owned(), + )); + for (index, arm) in arms.iter().enumerate() { + if index > 0 { + fragments.push(Fragment::Lit(", ".to_owned())); + } + if form == Form::ClassInfo && arm.is_none_literal_expr() { + fragments.push(Fragment::Lit("type(None)".to_owned())); + } else { + fragments.push(Fragment::Src(arm.range())); + } + } + // the tuple's trailing comma is what makes a one-arm union a tuple rather + // than a parenthesized class + fragments.push(Fragment::Lit( + match form { + Form::ClassInfo => ",)", + Form::Union => "]", + } + .to_owned(), + )); + fragments +} + +/// Flatten `a | b | c`, which parses as `(a | b) | c`, into its arms. +fn collect_arms<'ast>(expr: &'ast Expr, arms: &mut Vec<&'ast Expr>) { + if let Expr::BinOp(binop) = expr + && binop.op == Operator::BitOr + { + collect_arms(&binop.left, arms); + collect_arms(&binop.right, arms); + return; + } + arms.push(expr); +} + +#[cfg(test)] +mod tests { + use crate::{Config, PythonVersion, transpile}; + use indoc::indoc; + + /// transpile for a target that predates `type.__or__` + fn lowered(input: &str) -> String { + let config = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + transpile(input, &config).unwrap() + } + + /// a union assigned as a value builds a `types.UnionType`, which is what + /// 3.10 added; `typing.Union` is the spelling every version has + #[test] + fn an_alias_is_spelled_out() { + assert_eq!( + lowered("Alias = int | str\n"), + indoc! {" + from __future__ import annotations + from typing import Union + Alias = Union[int, str] + "} + ); + } + + /// `isinstance` rejects a `typing.Union` and accepts a tuple, so the + /// classinfo argument gets the other spelling + #[test] + fn isinstance_takes_a_tuple() { + let out = lowered("def f(x: object):\n return isinstance(x, int | str)\n"); + assert!(out.contains("isinstance(x, (int, str,))"), "got:\n{out}"); + assert!(!out.contains("Union"), "got:\n{out}"); + } + + /// `None` is shorthand the union operator understands and a tuple does not + #[test] + fn none_becomes_its_class_in_a_tuple() { + let out = lowered("def f(x: object):\n return isinstance(x, int | None)\n"); + assert!( + out.contains("isinstance(x, (int, type(None),))"), + "got:\n{out}" + ); + } + + /// a union nested in the tuple `isinstance` was already given is still a + /// tuple — nesting is something `isinstance` accepts + #[test] + fn a_nested_classinfo_union_is_a_tuple_too() { + let out = lowered("def f(x: object):\n return isinstance(x, (bytes, int | str))\n"); + assert!( + out.contains("isinstance(x, (bytes, (int, str,)))"), + "got:\n{out}" + ); + } + + /// a `cast` target is a type expression the runtime still evaluates + #[test] + fn a_cast_target_is_spelled_out() { + let out = lowered(indoc! {" + from typing import cast + def f(v: object): + return cast(int | str, v) + "}); + assert!(out.contains("cast(Union[int, str], v)"), "got:\n{out}"); + } + + /// an annotation is a string on every target this runs for, so it needs no + /// rewriting and keeps the spelling the author chose + #[test] + fn an_annotation_is_left_alone() { + let out = lowered("def f(x: int | str) -> bytes | None: ...\n"); + assert!(out.contains("x: int | str"), "got:\n{out}"); + assert!(out.contains("-> bytes | None"), "got:\n{out}"); + } + + /// an ordinary bitwise or is not a union whatever it is written between + #[test] + fn a_bitwise_or_is_untouched() { + let out = lowered("def f(a: int, b: int) -> int:\n return a | b\n"); + assert!(out.contains("return a | b"), "got:\n{out}"); + } + + /// a file that means something else by `isinstance` gets no special + /// treatment for its second argument + #[test] + fn a_shadowed_isinstance_is_not_a_classinfo_call() { + let out = lowered(indoc! {" + def isinstance(x: object, t: object) -> bool: + return True + + def f(x: object): + return isinstance(x, int | str) + "}); + assert!( + out.contains("isinstance(x, Union[int, str])"), + "got:\n{out}" + ); + } + + /// a target that has `type.__or__` keeps every union as written + #[test] + fn untouched_from_python_310() { + let out = transpile( + "Alias = int | str\ndef f(x: object):\n return isinstance(x, int | str)\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("Alias = int | str"), "got:\n{out}"); + assert!(out.contains("isinstance(x, int | str)"), "got:\n{out}"); + } +} diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 97ecbf5e16..b6f442e160 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -156,6 +156,14 @@ pub(crate) trait TypeInfo { /// reject as its classinfo argument at runtime fn is_keeps_identity(&self, expr: &Expr) -> bool; + /// whether `expr` is a PEP 604 union standing where the runtime will + /// evaluate it — `isinstance(x, int | str)`, a `cast` target, an alias + /// assigned at module level. `type.__or__` only arrived in python 3.10, so + /// below that the union has to be spelled another way; an annotation is not + /// one of these, since the lowering defers every annotation for such a + /// target and nothing ever evaluates it + fn is_runtime_union(&self, expr: &Expr) -> bool; + /// when `attribute` resolves to a basedpython `extension` member, the /// backing-function rewrite to apply (`xs.second()` → /// `_by_ext__list__second(xs)`). `None` for ordinary attributes — @@ -624,6 +632,22 @@ impl TypeInfo for SemanticModel<'_> { }) } + fn is_runtime_union(&self, expr: &Expr) -> bool { + // `x = int | str` reads as the `types.UnionType` object it builds. in a + // position that is *also* a type expression — a `cast` target, an + // `isinstance` classinfo — the same operator reads as a `TypeForm`: the + // type it denotes rather than the object it makes. both are evaluated + // at runtime, so both have to be lowered. + // + // every other reading is left alone, which is what keeps an ordinary + // `a | b` — and a class whose metaclass gives `|` a meaning of its own — + // out of this + matches!( + expr.inferred_type(self), + Some(Type::KnownInstance(KnownInstanceType::UnionType(_)) | Type::TypeForm(_)) + ) + } + fn extension_attribute_info( &self, attribute: &ruff_python_ast::ExprAttribute, diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 01ccd97ab5..2f15a62359 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -685,7 +685,7 @@ pub(crate) fn cmd_compile( )); let built = if emit_c_only { - by_build::emit_lowered(lowered, &source, &out_dir, &options) + by_build::emit_lowered(lowered, &source, &out_dir, &options, toolchain.version) } else { by_build::build_lowered(lowered, &source, &toolchain, &out_dir, &options).inspect( |built| { diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index c631cc6588..84d61c62f6 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -228,13 +228,10 @@ fn compile_transpiles_the_fallback_with_the_lowering_options_it_was_given() { // has always taken them; until this reached the cli there was no way to say so, // and every compile silently used the defaults // - // `except*` has no lowering, so this declines and the fallback is what runs + // an `async def` has no native lowering, so this declines and the fallback + // is what runs let source = "\ -def total(s: str, n: int) -> int: - try: - pass - except* ValueError: - pass +async def total(s: str, n: int) -> int: return len(s) + n "; let dir = std::env::temp_dir().join("by_cli_soundness"); diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 81fffb4fcf..967210800c 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -40,13 +40,25 @@ source (.by) ├─ phase 2c lazy-import marking │ └─ lower imports to the `lazy` keyword (3.15+) or a runtime polyfill │ + ├─ phase 2d version polyfill + │ └─ rewrite syntax the target python cannot parse. today that is the + │ `match` statement, lowered to an `if`/`elif` chain for a target below + │ 3.10. it runs over the *finished* python rather than over `.by`, so a + │ `match` an earlier lowering generated is lowered by the same code as + │ one the author wrote + │ └─ phase 3 syntax verification ├─ parse the final output as `.py` — any parse error aborts with a │ source-annotated diagnostic (the span is mapped back to `.by`) - └─ scan the AST for leftover basedpython-only flags - (`is_anon_named_tuple`, `is_anon_named_tuple_value`, `is_typeof`). - a leftover flag means a transform failed to lower its construct; the - pipeline aborts rather than emit syntactically-valid-but-wrong Python + ├─ scan the AST for leftover basedpython-only flags + │ (`is_anon_named_tuple`, `is_anon_named_tuple_value`, `is_typeof`). + │ a leftover flag means a transform failed to lower its construct; the + │ pipeline aborts rather than emit syntactically-valid-but-wrong Python + └─ parse it again *as the target version* and report any construct that + version cannot parse. the first check asks whether the output is + python at all; this one asks whether it is python the declared floor + can run, so syntax no polyfill covers is a diagnostic instead of a + `SyntaxError` at import time in generated code ``` entry points in `crates/by_transforms/src/lib.rs`: diff --git a/docs/basedpython/features/polyfills.md b/docs/basedpython/features/polyfills.md index df0e382fe1..d35668adb2 100644 --- a/docs/basedpython/features/polyfills.md +++ b/docs/basedpython/features/polyfills.md @@ -17,7 +17,17 @@ _V = TypeVar("_V") class Map(Generic[_K, _V]): ... ``` -basedpython backfills modern python syntax and stdlib features to older supported versions this way, at transpile time. minimum supported runtime is **python 3.10** +basedpython backfills modern python syntax and stdlib features to older supported versions this way, at transpile time. **python 3.10** is the version the toolchain is built around, and the one everything below is written against; the 3.10 section covers the constructs that are lowered for a target older still + +whatever is left — syntax no polyfill covers, aimed at a version that cannot parse it — is a transpile error rather than a file that fails at import. the output is parsed a second time as the target version, and the first construct that version does not have is reported against the `.by` line that produced it: + +```text +error[invalid-syntax]: Cannot use `except*` on Python 3.9 (syntax was added in Python 3.11) + --> probe.by:4:1 + | +4 | except* ValueError: + | ^^^^^^^^^^^^^^^^^^^^^^^ +``` scope of this page: rewrites that apply to **plain python source** (forms a user could type into a `.py` file). basedpython-specific surface syntax has its own feature page @@ -311,6 +321,62 @@ except ModuleNotFoundError: ______________________________________________________________________ +## Python 3.10 + +### the `match` statement (PEP 634) + +`match` is grammar, so a target below 3.10 cannot even parse a file containing one. every `match` in the output — the ones you wrote, and the ones other lowerings produce for [`let` destructuring](destructuring.md), [`if let`](if-let.md), [statement expressions](statement-expressions.md) and [enum](enums.md) exhaustiveness — becomes an `if`/`elif` chain whose conditions do the matching: + +```python +# python source +match point: + case Point(0, y) if y > 0: + north(y) + case _: + elsewhere() +``` + +```python +# generated Python +if [__by_match_0__ := (point)]: + if isinstance(__by_match_0__, (__by_match_1__ := (Point))) and ...: + north(y) + else: + elsewhere() +``` + +captures are bound by assignment expressions along the way, and the structure python cannot ask for in an expression — whether a value counts as a sequence or a mapping, what a class's `__match_args__` names — is asked of small helper functions the output carries. the subject is evaluated once, the cases are tried in order, and a sub-pattern that fails falls through to the next case exactly as it would have + +only the `match` and `case` headers are replaced, so every case body keeps its source bytes at its own indentation and the statement occupies the same lines it did before + +two things are not reproduced: a comment written *inside* a multi-line pattern is dropped, and a temporary is left bound after the statement (python has no expression that unbinds a name). the temporaries are dunder-named, so a `match` in a class body leaves nothing `enum` or `dataclass` reads as a member + +the lowering binds with assignment expressions, so it needs python 3.8. below that a `match` is reported rather than lowered + +### `X | Y` at runtime (PEP 604) + +`int | str` calls `type.__or__`, which arrived in 3.10. in an *annotation* that costs nothing — a target this old always gets `from __future__ import annotations`, so no annotation is ever evaluated — but where the value is really produced it is a `TypeError` at import time. those are spelled the way the target can: + +```python +# python source +Alias = int | str +isinstance(x, int | None) +cast(int | str, value) +``` + +```python +# generated Python +Alias = Union[int, str] +isinstance(x, (int, type(None),)) +cast(Union[int, str], value) +``` + +the two spellings are not interchangeable — `isinstance` takes a tuple of classes and rejects a `typing.Union` — so the classinfo argument of `isinstance` and `issubclass` becomes a tuple and everything else a `Union`. whether a `|` is a union at all is asked of the checker rather than guessed from the shape, so an ordinary bitwise or is left alone + +[`T?`](wrapped-results.md) is spelled `Union[T, None]` on these targets for the same reason + +______________________________________________________________________ + ## generic classes and functions (PEP 695) python 3.12 introduced compact generic syntax. basedpython rewrites it using `typing.TypeVar` and `typing.Generic`