lower the match statement and runtime unions for older pythons, and report what neither covers

crates/by_build/src/lib.rs

@@ -76,8 +76,9 @@ pub fn emit_lowered(
     source: &str,
     out_dir: &Path,
     options: &Options,
+    version: Option<(u8, u8)>,
 ) -> Result<Built> {
-    let module = finish(module, source, options, None)?;
+    let module = finish(module, source, options, version)?;
     emit_verified(&module, out_dir, options)
 }
 
@@ -90,8 +91,9 @@ pub fn emit_source(
     module_name: impl Into<by_ir::ModuleName>,
     out_dir: &Path,
     options: &Options,
+    version: Option<(u8, u8)>,
 ) -> Result<Built> {
-    let module = lower(source, module_name, options, None)?;
+    let module = lower(source, module_name, options, version)?;
     emit_verified(&module, out_dir, options)
 }
 

crates/by_build/tests/differential.rs

@@ -540,11 +540,18 @@ fn agree_in(
     let module = format!("by_diff_{tag}");
 
     // the interpreted leg: for basedpython, the transpiler's own output run by
-    // cpython, under the same config `by_build` uses, so the two legs are the same
-    // program. for python there is nothing to transpile — it already is one
+    // cpython, under the same config `by_build` uses — including the *target
+    // version*, which has to be this interpreter's or the two legs are not the
+    // same program. for python there is nothing to transpile — it already is one
     let interpreted_source = match language {
         by_irbuild::Language::BasedPython => {
-            by_transforms::transpile(source, &Config::default()).expect("the source transpiles")
+            let mut config = Config::default();
+            if let Some((major, minor)) = toolchain.version
+                && let Ok(parsed) = format!("{major}.{minor}").parse()
+            {
+                config.min_version = parsed;
+            }
+            by_transforms::transpile(source, &config).expect("the source transpiles")
         }
         by_irbuild::Language::Python => source.to_string(),
     };

crates/by_build/tests/end_to_end.rs

@@ -570,6 +570,7 @@ def kept(a: str, b: str) -> object:
         "by_e2e_append",
         &dir,
         &Options::default(),
+        None,
     )
     .expect("the module emits");
     let emitted = std::fs::read_to_string(&built.artifact.source).expect("the C is readable");
@@ -1097,8 +1098,14 @@ fn the_emitted_c_names_no_pointer_type_it_does_not_mean() {
     }
     let dir = std::env::temp_dir().join("by_e2e_pointers");
     let _ = std::fs::remove_dir_all(&dir);
-    let built = by_build::emit_source(POINTER_SOURCE, "by_e2e_pointers", &dir, &Options::default())
-        .expect("the module emits");
+    let built = by_build::emit_source(
+        POINTER_SOURCE,
+        "by_e2e_pointers",
+        &dir,
+        &Options::default(),
+        None,
+    )
+    .expect("the module emits");
 
     let object = dir.join("by_e2e_pointers.o");
     let mut args = by_build::compile_command(&toolchain, &built.artifact.source, &object, &dir);

crates/by_transforms/src/lib.rs

@@ -201,9 +201,11 @@ pub fn transpile(source: &str, config: &Config) -> Result<String, String> {
     let final_output = run_anon_named_tuple_cleanup(final_output, config)?;
     let final_output =
         run_lazy_import_phase(final_output, config, &model.eagerly_imported_modules());
+    let final_output = run_version_polyfill_phase(final_output, config);
 
     // --- Phase 3: syntax verification ---
     verify_syntax(&final_output).map_err(|e| e.message)?;
+    verify_target_syntax(&final_output, config).map_err(|e| e.message)?;
 
     Ok(final_output)
 }
@@ -365,6 +367,7 @@ pub fn transpile_typed_with_map(
     let final_output = run_import_redirect_phase(output, config);
     let final_output = run_anon_named_tuple_cleanup(final_output, config)?;
     let final_output = run_lazy_import_phase(final_output, config, &eager_imports);
+    let final_output = run_version_polyfill_phase(final_output, config);
 
     // phases 1-2c only prepend preambles at the top and edit within lines, so
     // the spliced body keeps its line correspondence: prepend one `None` per
@@ -398,7 +401,9 @@ pub fn transpile_typed_with_map(
     line_map.extend(composed[kept..].iter().copied());
 
     // verify last: on failure, map the generated span back to a `.by` range
-    if let Err(mut err) = verify_syntax(&final_output) {
+    let verified =
+        verify_syntax(&final_output).and_then(|()| verify_target_syntax(&final_output, config));
+    if let Err(mut err) = verified {
         err.by_range = err.output_range.and_then(|r| {
             output_offset_to_by_range(&line_map, &final_output, original_source, r.start())
         });
@@ -571,6 +576,43 @@ fn run_lazy_import_phase(source: String, config: &Config, eager: &[String]) -> S
     }
 }
 
+/// Version polyfill phase: rewrite syntax the target python cannot parse into
+/// syntax it can. Runs over the finished python rather than over `.by`, so a
+/// `match` an earlier lowering *generated* — for a `let` destructuring, an
+/// `if let`, a statement expression — is lowered by the same code as one the
+/// author wrote.
+///
+/// Nothing here changes how many lines the file has: the polyfill rewrites
+/// headers in place and pads them back to their original height, so the only
+/// lines it adds are the runtime preamble's, at the top, where the line map
+/// already accounts for generated leading lines.
+fn run_version_polyfill_phase(source: String, config: &Config) -> String {
+    transforms::match_polyfill::lower(source, config.min_version)
+}
+
+/// Re-parse the transpiled output *as the target python version* and report any
+/// construct that version cannot parse.
+///
+/// [`verify_syntax`] asks whether the output is python at all; this asks whether
+/// it is python the file's declared floor can run. Without it, syntax no
+/// polyfill covers — `except*`, t-strings, a PEP 701 f-string — is emitted
+/// verbatim and fails at import time in generated code the author never wrote.
+fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileError> {
+    let options = ruff_python_parser::ParseOptions::from(ruff_python_ast::PySourceType::Python)
+        .with_target_version(config.min_version);
+    let parsed = ruff_python_parser::parse_unchecked(source, options);
+    let Some(first) = parsed.unsupported_syntax_errors().first() else {
+        return Ok(());
+    };
+    // the parser's own wording already names both versions: "Cannot use
+    // `except*` on Python 3.9 (syntax was added in Python 3.11)"
+    Err(TranspileError {
+        message: first.to_string(),
+        output_range: Some(first.range),
+        by_range: None,
+    })
+}
+
 /// Splice `preamble` into `body` where generated lines belong: after the module
 /// docstring and any `from __future__ import`, each of which is only valid
 /// where it already is.

crates/by_transforms/src/transforms/annotation.rs

@@ -1,12 +1,23 @@
 use ruff_diagnostics::{Edit, Fix};
-use ruff_python_ast::{Expr, Stmt};
+use ruff_python_ast::{Expr, PythonVersion, Stmt};
 use ruff_text_size::Ranged;
 
+use crate::Config;
 use crate::transforms::ast_driver::{PassContext, TypeAwarePass};
 use crate::transforms::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions};
 use crate::transforms::{literal_types, optional_type};
 use crate::type_info::TypeInfo;
 
+/// The element type an unpacked tuple element wraps, whichever way the target
+/// spells the unpack.
+fn strip_unpack(element: &str) -> Option<&str> {
+    element.strip_prefix('*').or_else(|| {
+        element
+            .strip_prefix("Unpack[")
+            .and_then(|rest| rest.strip_suffix(']'))
+    })
+}
+
 /// Rewrites tuple literal types in type positions.
 ///
 /// `a: (int, str)` → `a: tuple[int, str]`
@@ -18,18 +29,40 @@ use crate::type_info::TypeInfo;
 pub(crate) struct TupleLiteralType<'src> {
     source: &'src str,
     types: &'src dyn TypeInfo,
+    min_version: PythonVersion,
+    /// set when a lowering spelled an `Unpack`, so the pass can ask for the import
+    pub(crate) needs_unpack_import: std::cell::Cell<bool>,
     pub(crate) edits: Vec<Fix>,
 }
 
 impl<'src> TupleLiteralType<'src> {
-    pub(crate) fn new(source: &'src str, types: &'src dyn TypeInfo) -> Self {
+    pub(crate) fn new(
+        source: &'src str,
+        types: &'src dyn TypeInfo,
+        min_version: PythonVersion,
+    ) -> Self {
         Self {
             source,
             types,
+            min_version,
+            needs_unpack_import: std::cell::Cell::new(false),
             edits: Vec::new(),
         }
     }
 
+    /// How an unpacked element of a tuple type is spelled for the target. The
+    /// star form is PEP 646, python 3.11; below that it is a *syntax* error
+    /// rather than something the `__future__` import could defer, so a tuple
+    /// type has to say `Unpack[...]` instead.
+    fn unpack(&self, inner: &str) -> String {
+        if self.min_version >= PythonVersion::PY311 {
+            format!("*{inner}")
+        } else {
+            self.needs_unpack_import.set(true);
+            format!("Unpack[{inner}]")
+        }
+    }
+
     fn src(&self, range: ruff_text_size::TextRange) -> &str {
         &self.source[usize::from(range.start())..usize::from(range.end())]
     }
@@ -75,7 +108,7 @@ impl<'src> TupleLiteralType<'src> {
                 // rather than the wrapped `tuple[*tuple[T, ...]]` form
                 if parameter_shape
                     && lowered.len() == 1
-                    && let Some(rest) = lowered[0].strip_prefix("*")
+                    && let Some(rest) = strip_unpack(&lowered[0])
                 {
                     return Some(rest.to_owned());
                 }
@@ -218,7 +251,7 @@ impl<'src> TupleLiteralType<'src> {
                     let value_src = self
                         .transform_annotation(&named.value)
                         .unwrap_or_else(|| self.src(named.value.range()).to_owned());
-                    return format!("*tuple[{value_src}, ...]");
+                    return self.unpack(&format!("tuple[{value_src}, ...]"));
                 }
                 self.transform_annotation(&named.value)
                     .unwrap_or_else(|| self.src(named.value.range()).to_owned())
@@ -233,17 +266,17 @@ impl<'src> TupleLiteralType<'src> {
                     .transform_annotation(&s.value)
                     .unwrap_or_else(|| self.src(s.value.range()).to_owned());
                 if parameter_shape {
-                    format!("*tuple[{value_src}, ...]")
+                    self.unpack(&format!("tuple[{value_src}, ...]"))
                 } else {
-                    format!("*{value_src}")
+                    self.unpack(&value_src)
                 }
             }
             // a plain element type — also lower a nested `?` (`(int, str?)`),
             // which `transform_annotation` doesn't handle. element-scoped, so the
             // tuple's whole-expression edit subsumes the optional pass's edit
             _ => self
                 .transform_annotation(elt)
-                .or_else(|| optional_type::rewrite_type_expr(self.source, elt))
+                .or_else(|| optional_type::rewrite_type_expr(self.source, elt, self.min_version))
                 .unwrap_or_else(|| self.src(elt.range()).to_owned()),
         }
     }
@@ -275,17 +308,18 @@ impl TypeExprVisitor for TupleLiteralType<'_> {
 
 pub(crate) struct TupleLiteralTypePass<'src> {
     source: &'src str,
+    config: Config,
 }
 
 impl<'src> TupleLiteralTypePass<'src> {
-    pub(crate) fn new(source: &'src str) -> Self {
-        Self { source }
+    pub(crate) fn new(source: &'src str, config: Config) -> Self {
+        Self { source, config }
     }
 }
 
 impl TypeAwarePass for TupleLiteralTypePass<'_> {
     fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) {
-        let mut inner = TupleLiteralType::new(self.source, types);
+        let mut inner = TupleLiteralType::new(self.source, types, self.config.min_version);
         walk_type_positions(stmts, Some(types), &mut inner);
         let mut wraps_literal = false;
         for fix in inner.edits {
@@ -298,6 +332,10 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> {
                 ctx.text_edits.push((range, repl));
             }
         }
+        if inner.needs_unpack_import.get() {
+            ctx.required_imports
+                .push("from typing import Unpack".to_owned());
+        }
         // when our embedded literal-type lowering produced `Literal[...]` text,
         // request the import. the standalone literal_types pass doesn't see
         // the bare literal anymore because we've replaced its parent annotation
@@ -311,7 +349,7 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> {
 #[cfg(test)]
 mod tests {
     use crate::python_passthrough::unchanged;
-    use crate::{Config, transpile};
+    use crate::{Config, PythonVersion, transpile};
     use indoc::indoc;
 
     fn check(input: &str, expected: &str) {
@@ -321,6 +359,18 @@ mod tests {
         );
     }
 
+    /// the same, against a target that has PEP 646
+    fn check_at(min_version: PythonVersion, input: &str, expected: &str) {
+        let config = Config {
+            min_version,
+            ..Config::test_default()
+        };
+        assert_eq!(
+            transpile(input, &config).unwrap(),
+            crate::python_passthrough::lazify_expected(expected)
+        );
+    }
+
     #[test]
     fn simple_tuple_annotation() {
         check("a: (int, str)\n", "a: tuple[int, str]\n");
@@ -331,21 +381,39 @@ mod tests {
         check("a: (int,)\n", "a: tuple[int]\n");
     }
 
-    /// a bare `*A` splices `A` in — python spells that the same way, unlike the
-    /// `*: T` variadic it shares an AST shape with
+    /// a bare `*A` splices `A` in — python spells that the same way from 3.11,
+    /// unlike the `*: T` variadic it shares an AST shape with
     #[test]
     fn unpacked_element() {
-        check("a: (int, *A)\n", "a: tuple[int, *A]\n");
+        check_at(
+            PythonVersion::PY311,
+            "a: (int, *A)\n",
+            "a: tuple[int, *A]\n",
+        );
+    }
+
+    /// a star in a subscript is 3.11 grammar, so a `__future__` import cannot
+    /// defer it the way it defers an annotation's *evaluation* — below 3.11 the
+    /// unpack has to be spelled out
+    #[test]
+    fn unpacked_element_before_pep_646() {
+        check(
+            "a: (int, *A)\n",
+            indoc! {"
+                from typing_extensions import Unpack
+                a: tuple[int, Unpack[A]]
+            "},
+        );
     }
 
     #[test]
     fn lone_unpacked_element_without_a_comma() {
-        check("a: (*A)\n", "a: tuple[*A]\n");
+        check_at(PythonVersion::PY311, "a: (*A)\n", "a: tuple[*A]\n");
     }
 
     #[test]
     fn lone_unpacked_element_with_a_comma() {
-        check("a: (*A,)\n", "a: tuple[*A]\n");
+        check_at(PythonVersion::PY311, "a: (*A,)\n", "a: tuple[*A]\n");
     }
 
     #[test]
@@ -458,15 +526,27 @@ mod tests {
     fn variadic_tuple_annotation() {
         // `*: T` in a tuple type expands to `*tuple[T, ...]` so the tuple
         // can hold zero+ values of T after the leading positional fields
-        check("b: (int, *: str)\n", "b: tuple[int, *tuple[str, ...]]\n");
+        check_at(
+            PythonVersion::PY311,
+            "b: (int, *: str)\n",
+            "b: tuple[int, *tuple[str, ...]]\n",
+        );
+        check(
+            "b: (int, *: str)\n",
+            indoc! {"
+                from typing_extensions import Unpack
+                b: tuple[int, Unpack[tuple[str, ...]]]
+            "},
+        );
     }
 
     #[test]
     fn named_variadic_tuple_annotation() {
         // `*name: T` in a tuple type behaves the same as `*: T` — the name
         // is metadata for callable-parameter use and has no effect on tuple
         // type semantics
-        check(
+        check_at(
+            PythonVersion::PY311,
             "b: (int, *args: str)\n",
             "b: tuple[int, *tuple[str, ...]]\n",
         );