output-position widening for invariant builtin containers

fresh-copy methods of invariant containers (list/set/dict) return a brand new
object the caller owns, so widening its element type at the call site is sound —
but `list[int]` isn't a `list[int | None]`, so today `a.copy()` can't be
assigned to a wider specialization.

encode the fix as a `Never`-defaulted output type parameter unioned into each
invariant return position: `def copy[Widen = Never](self) -> list[Element |
Widen]`. with an expected type it solves to the widening; without one it
defaults to `Never` and collapses back, so inference is unchanged.

new `output-widening` typeshed patch (post-pep695 pass, keyed on `in out`
variance so covariant frozenset/tuple are left alone) + regenerated
builtins.byi. it widens only methods reached by an ordinary call — `copy`,
`set.union`/`difference`/`intersection`/`symmetric_difference` — where the
caller's expected type already flows into inference. operator dunders
(`__add__`, `__getitem__`, ...) are deliberately not widened: making their
returns widen would require bidirectional inference on every binary op and
subscript, which is far too expensive on real code (it times out xarray).

Author: KotlinIsland

crates/by_typeshed_patch/src/lib.rs

@@ -42,13 +42,39 @@ pub struct Edit {
     pub replacement: String,
 }
 
-/// registry of every patch the sync pipeline must apply, in declared order
+/// registry of every legacy-form patch the sync pipeline must apply, in
+/// declared order. these run before the pep 695 conversion and see the legacy
+/// `TypeVar` + `Generic[...]` form
 pub fn all_patches() -> Vec<Box<dyn Patch>> {
     // patches are added here as upstream syncs surface concrete drift. each
     // entry must have a corresponding module in `src/patches/` with tests
     vec![Box::new(patches::mapping::MappingKeyCovariance)]
 }
 
+/// registry of patches that run *after* the pep 695 conversion, over the final
+/// form with explicit variance keywords. a patch belongs here when it needs the
+/// resolved variance (`in out` vs `out`) that only the converted form exposes
+pub fn all_post_patches() -> Vec<Box<dyn Patch>> {
+    vec![Box::new(patches::output_widening::OutputWidening)]
+}
+
+/// dotted module name for a typeshed file path relative to `stdlib/`, e.g.
+/// `typing.byi` -> `typing`, `os/path.byi` -> `os.path`,
+/// `asyncio/__init__.byi` -> `asyncio`
+pub(crate) fn module_qualname(path: &Path) -> Option<String> {
+    let stem = path.file_stem()?.to_str()?;
+    let mut parts: Vec<&str> = path
+        .parent()
+        .into_iter()
+        .flat_map(Path::components)
+        .filter_map(|component| component.as_os_str().to_str())
+        .collect();
+    if stem != "__init__" {
+        parts.push(stem);
+    }
+    Some(parts.join("."))
+}
+
 /// apply `edits` to `source`, returning the new text. edits must be disjoint;
 /// applied in reverse start order so earlier offsets remain valid
 pub fn apply_edits(source: &str, mut edits: Vec<Edit>) -> String {

crates/by_typeshed_patch/src/main.rs

@@ -1,17 +1,18 @@
 //! binary entry: walks the basedpython typeshed and rewrites each `.byi`
 //! stub. invoked by `scripts/sync_typeshed_by.sh` after reverse-transpile
 //!
-//! each file is rewritten in two passes:
+//! each file is rewritten in three passes:
 //!
 //! 1. the registered semantic [`Patch`]es (e.g. mapping key covariance), which
 //!    operate on the legacy `TypeVar` + `Generic[...]` form
 //! 1. the pep 695 conversion ([`by_typeshed_patch::pep695`]), which turns
 //!    legacy generic classes into pep 695 headers with explicit variance and
 //!    nice names
+//! 1. the post-conversion [`Patch`]es (e.g. output widening), which need the
+//!    explicit variance keywords the conversion emits
 //!
-//! the passes run sequentially with a re-parse in between: a patch may rewrite
-//! a typevar reference (covariance) that the conversion then renames, so the
-//! conversion must see the patched source
+//! the passes run sequentially with a re-parse between each: a pass may rewrite
+//! a reference the next pass depends on, so each must see the prior output
 //!
 //! usage:
 //!   `by_typeshed_patch` `<typeshed-stdlib-dir>`
@@ -27,7 +28,7 @@ use ruff_python_ast::PySourceType;
 use ruff_python_parser::parse_unchecked_source;
 use walkdir::WalkDir;
 
-use by_typeshed_patch::{Patch, all_patches, apply_edits, pep695};
+use by_typeshed_patch::{Patch, all_patches, all_post_patches, apply_edits, pep695};
 
 fn main() -> ExitCode {
     match run() {
@@ -50,7 +51,8 @@ fn run() -> Result<()> {
     }
 
     let patches = all_patches();
-    if patches.is_empty() {
+    let post_patches = all_post_patches();
+    if patches.is_empty() && post_patches.is_empty() {
         eprintln!("no patches registered; nothing to do");
         return Ok(());
     }
@@ -64,7 +66,7 @@ fn run() -> Result<()> {
         }
         visited += 1;
         let rel = path.strip_prefix(&root).unwrap_or(path);
-        if apply_patches_to_file(path, rel, &patches)
+        if apply_patches_to_file(path, rel, &patches, &post_patches)
             .with_context(|| format!("applying patches to {}", path.display()))?
         {
             patched += 1;
@@ -74,7 +76,12 @@ fn run() -> Result<()> {
     Ok(())
 }
 
-fn apply_patches_to_file(path: &Path, rel: &Path, patches: &[Box<dyn Patch>]) -> Result<bool> {
+fn apply_patches_to_file(
+    path: &Path,
+    rel: &Path,
+    patches: &[Box<dyn Patch>],
+    post_patches: &[Box<dyn Patch>],
+) -> Result<bool> {
     let original = fs::read_to_string(path).with_context(|| format!("{}", path.display()))?;
 
     // pass 1: registered semantic patches over the legacy form
@@ -98,12 +105,25 @@ fn apply_patches_to_file(path: &Path, rel: &Path, patches: &[Box<dyn Patch>]) ->
     // any typevar references the patches rewrote)
     let reparsed = parse_unchecked_source(&patched, PySourceType::BasedPythonStub);
     let conversion = pep695::convert_module(&reparsed, &patched);
-    let final_source = if conversion.is_empty() {
+    let converted = if conversion.is_empty() {
         patched
     } else {
         apply_edits(&patched, conversion)
     };
 
+    // pass 3: post-conversion patches over the final pep 695 form (re-parsed so
+    // they see the explicit variance keywords the conversion emitted)
+    let reparsed = parse_unchecked_source(&converted, PySourceType::BasedPythonStub);
+    let mut post_edits = Vec::new();
+    for patch in post_patches {
+        post_edits.extend(patch.rewrite(rel, &reparsed, &converted));
+    }
+    let final_source = if post_edits.is_empty() {
+        converted
+    } else {
+        apply_edits(&converted, post_edits)
+    };
+
     if final_source == original {
         return Ok(false);
     }

crates/by_typeshed_patch/src/patches/mapping.rs

@@ -18,7 +18,7 @@ use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk
 use ruff_python_ast::{Expr, ModModule, Stmt};
 use ruff_python_parser::Parsed;
 
-use crate::{Edit, Patch};
+use crate::{Edit, Patch, module_qualname};
 
 /// module that owns the canonical `Mapping` definition
 const MODULE: &str = "typing";
@@ -97,23 +97,6 @@ impl<'a> SourceOrderVisitor<'a> for MappingKeyReferences {
     }
 }
 
-/// dotted module name for a typeshed file path relative to `stdlib/`, e.g.
-/// `typing.byi` -> `typing`, `os/path.byi` -> `os.path`,
-/// `asyncio/__init__.byi` -> `asyncio`
-fn module_qualname(path: &Path) -> Option<String> {
-    let stem = path.file_stem()?.to_str()?;
-    let mut parts: Vec<&str> = path
-        .parent()
-        .into_iter()
-        .flat_map(Path::components)
-        .filter_map(|component| component.as_os_str().to_str())
-        .collect();
-    if stem != "__init__" {
-        parts.push(stem);
-    }
-    Some(parts.join("."))
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/by_typeshed_patch/src/patches/mod.rs

@@ -2,3 +2,4 @@
 //! each in `all_patches()` in the crate root
 
 pub mod mapping;
+pub mod output_widening;