diff --git a/crates/by_typeshed_patch/src/lib.rs b/crates/by_typeshed_patch/src/lib.rs index f3b1b479f7..988f26d52e 100644 --- a/crates/by_typeshed_patch/src/lib.rs +++ b/crates/by_typeshed_patch/src/lib.rs @@ -42,13 +42,39 @@ pub struct Edit { pub replacement: String, } -/// registry of every patch the sync pipeline must apply, in declared order +/// registry of every legacy-form patch the sync pipeline must apply, in +/// declared order. these run before the pep 695 conversion and see the legacy +/// `TypeVar` + `Generic[...]` form pub fn all_patches() -> Vec> { // patches are added here as upstream syncs surface concrete drift. each // entry must have a corresponding module in `src/patches/` with tests vec![Box::new(patches::mapping::MappingKeyCovariance)] } +/// registry of patches that run *after* the pep 695 conversion, over the final +/// form with explicit variance keywords. a patch belongs here when it needs the +/// resolved variance (`in out` vs `out`) that only the converted form exposes +pub fn all_post_patches() -> Vec> { + vec![Box::new(patches::output_widening::OutputWidening)] +} + +/// dotted module name for a typeshed file path relative to `stdlib/`, e.g. +/// `typing.byi` -> `typing`, `os/path.byi` -> `os.path`, +/// `asyncio/__init__.byi` -> `asyncio` +pub(crate) fn module_qualname(path: &Path) -> Option { + let stem = path.file_stem()?.to_str()?; + let mut parts: Vec<&str> = path + .parent() + .into_iter() + .flat_map(Path::components) + .filter_map(|component| component.as_os_str().to_str()) + .collect(); + if stem != "__init__" { + parts.push(stem); + } + Some(parts.join(".")) +} + /// apply `edits` to `source`, returning the new text. edits must be disjoint; /// applied in reverse start order so earlier offsets remain valid pub fn apply_edits(source: &str, mut edits: Vec) -> String { diff --git a/crates/by_typeshed_patch/src/main.rs b/crates/by_typeshed_patch/src/main.rs index 931d828fbb..d9b5c29514 100644 --- a/crates/by_typeshed_patch/src/main.rs +++ b/crates/by_typeshed_patch/src/main.rs @@ -1,17 +1,18 @@ //! binary entry: walks the basedpython typeshed and rewrites each `.byi` //! stub. invoked by `scripts/sync_typeshed_by.sh` after reverse-transpile //! -//! each file is rewritten in two passes: +//! each file is rewritten in three passes: //! //! 1. the registered semantic [`Patch`]es (e.g. mapping key covariance), which //! operate on the legacy `TypeVar` + `Generic[...]` form //! 1. the pep 695 conversion ([`by_typeshed_patch::pep695`]), which turns //! legacy generic classes into pep 695 headers with explicit variance and //! nice names +//! 1. the post-conversion [`Patch`]es (e.g. output widening), which need the +//! explicit variance keywords the conversion emits //! -//! the passes run sequentially with a re-parse in between: a patch may rewrite -//! a typevar reference (covariance) that the conversion then renames, so the -//! conversion must see the patched source +//! the passes run sequentially with a re-parse between each: a pass may rewrite +//! a reference the next pass depends on, so each must see the prior output //! //! usage: //! `by_typeshed_patch` `` @@ -27,7 +28,7 @@ use ruff_python_ast::PySourceType; use ruff_python_parser::parse_unchecked_source; use walkdir::WalkDir; -use by_typeshed_patch::{Patch, all_patches, apply_edits, pep695}; +use by_typeshed_patch::{Patch, all_patches, all_post_patches, apply_edits, pep695}; fn main() -> ExitCode { match run() { @@ -50,7 +51,8 @@ fn run() -> Result<()> { } let patches = all_patches(); - if patches.is_empty() { + let post_patches = all_post_patches(); + if patches.is_empty() && post_patches.is_empty() { eprintln!("no patches registered; nothing to do"); return Ok(()); } @@ -64,7 +66,7 @@ fn run() -> Result<()> { } visited += 1; let rel = path.strip_prefix(&root).unwrap_or(path); - if apply_patches_to_file(path, rel, &patches) + if apply_patches_to_file(path, rel, &patches, &post_patches) .with_context(|| format!("applying patches to {}", path.display()))? { patched += 1; @@ -74,7 +76,12 @@ fn run() -> Result<()> { Ok(()) } -fn apply_patches_to_file(path: &Path, rel: &Path, patches: &[Box]) -> Result { +fn apply_patches_to_file( + path: &Path, + rel: &Path, + patches: &[Box], + post_patches: &[Box], +) -> Result { let original = fs::read_to_string(path).with_context(|| format!("{}", path.display()))?; // pass 1: registered semantic patches over the legacy form @@ -98,12 +105,25 @@ fn apply_patches_to_file(path: &Path, rel: &Path, patches: &[Box]) -> // any typevar references the patches rewrote) let reparsed = parse_unchecked_source(&patched, PySourceType::BasedPythonStub); let conversion = pep695::convert_module(&reparsed, &patched); - let final_source = if conversion.is_empty() { + let converted = if conversion.is_empty() { patched } else { apply_edits(&patched, conversion) }; + // pass 3: post-conversion patches over the final pep 695 form (re-parsed so + // they see the explicit variance keywords the conversion emitted) + let reparsed = parse_unchecked_source(&converted, PySourceType::BasedPythonStub); + let mut post_edits = Vec::new(); + for patch in post_patches { + post_edits.extend(patch.rewrite(rel, &reparsed, &converted)); + } + let final_source = if post_edits.is_empty() { + converted + } else { + apply_edits(&converted, post_edits) + }; + if final_source == original { return Ok(false); } diff --git a/crates/by_typeshed_patch/src/patches/mapping.rs b/crates/by_typeshed_patch/src/patches/mapping.rs index 78ca6e0f95..d115441ce5 100644 --- a/crates/by_typeshed_patch/src/patches/mapping.rs +++ b/crates/by_typeshed_patch/src/patches/mapping.rs @@ -18,7 +18,7 @@ use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr, walk use ruff_python_ast::{Expr, ModModule, Stmt}; use ruff_python_parser::Parsed; -use crate::{Edit, Patch}; +use crate::{Edit, Patch, module_qualname}; /// module that owns the canonical `Mapping` definition const MODULE: &str = "typing"; @@ -97,23 +97,6 @@ impl<'a> SourceOrderVisitor<'a> for MappingKeyReferences { } } -/// dotted module name for a typeshed file path relative to `stdlib/`, e.g. -/// `typing.byi` -> `typing`, `os/path.byi` -> `os.path`, -/// `asyncio/__init__.byi` -> `asyncio` -fn module_qualname(path: &Path) -> Option { - let stem = path.file_stem()?.to_str()?; - let mut parts: Vec<&str> = path - .parent() - .into_iter() - .flat_map(Path::components) - .filter_map(|component| component.as_os_str().to_str()) - .collect(); - if stem != "__init__" { - parts.push(stem); - } - Some(parts.join(".")) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/by_typeshed_patch/src/patches/mod.rs b/crates/by_typeshed_patch/src/patches/mod.rs index 25bf14eb50..dc29ba0a4f 100644 --- a/crates/by_typeshed_patch/src/patches/mod.rs +++ b/crates/by_typeshed_patch/src/patches/mod.rs @@ -2,3 +2,4 @@ //! each in `all_patches()` in the crate root pub mod mapping; +pub mod output_widening; diff --git a/crates/by_typeshed_patch/src/patches/output_widening.rs b/crates/by_typeshed_patch/src/patches/output_widening.rs new file mode 100644 index 0000000000..c054a47bac --- /dev/null +++ b/crates/by_typeshed_patch/src/patches/output_widening.rs @@ -0,0 +1,467 @@ +//! output-position widening for the invariant builtin containers +//! +//! an invariant generic container cannot be assigned to a wider specialization: +//! `list[int]` is not a `list[int | None]`, because a caller holding the wider +//! type could insert a `None` the original never expected. but a method that +//! returns a *fresh* container of the same class — `list.copy`, `dict.copy`, the +//! set algebra (`union`, `difference`, ...) — hands back a brand new object the +//! caller solely owns, so widening its element type at the call site is sound +//! +//! this patch encodes that by giving each such method a `Never`-defaulted type +//! parameter and unioning it into every invariant position of the return type: +//! +//! ```by +//! def copy[Widen... = Never](self) -> list[Element | Widen...] +//! ``` +//! +//! with an expected type the parameter solves to the widening +//! (`b: list[int | None] = a.copy()`); with no expected type it defaults to +//! `Never` and `Element | Never` collapses back to `Element`, so ordinary +//! inference is unchanged (`reveal_type(a.copy())` is still `list[int]`) +//! +//! only methods reached by an ordinary call are widened, never operator dunders +//! (`__add__`, `__getitem__`, ...): a call already threads the caller's expected +//! type into inference, whereas widening an operator's return would need +//! bidirectional inference on every binary op / subscript, which is far too +//! expensive on real code. see `is_dunder` +//! +//! unlike the legacy-form semantic patches this runs in the post-pep 695 pass: +//! it keys off the explicit variance keywords to widen only invariant positions, +//! leaving covariant containers (`frozenset`, `tuple`) alone — their copies +//! already widen for free + +use std::collections::HashSet; +use std::path::Path; + +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_ast::{Expr, ModModule, Stmt, StmtClassDef, StmtFunctionDef, TypeParam, Variance}; +use ruff_python_parser::Parsed; +use ruff_text_size::Ranged; + +use crate::{Edit, Patch, module_qualname}; + +/// module that owns the container definitions we widen +const MODULE: &str = "builtins"; + +/// the mutable, invariant builtin containers. `frozenset` and `tuple` are +/// immutable and therefore covariant, so their copies already widen without help +const TARGET_CLASSES: &[&str] = &["list", "set", "dict"]; + +/// `Never` is version-guarded in `typing` (3.11+) but unconditional in +/// `typing_extensions`; builtins loads for every version, so we source it there +const NEVER_IMPORT_FROM: &str = "typing_extensions"; + +pub struct OutputWidening; + +impl Patch for OutputWidening { + fn name(&self) -> &'static str { + "output-widening" + } + + fn target_symbols(&self) -> &'static [&'static str] { + &["builtins.list", "builtins.set", "builtins.dict"] + } + + fn rewrite(&self, module_path: &Path, parsed: &Parsed, _source: &str) -> Vec { + if module_qualname(module_path).as_deref() != Some(MODULE) { + return Vec::new(); + } + + let body = &parsed.syntax().body; + let mut edits = Vec::new(); + for stmt in body { + if let Stmt::ClassDef(class) = stmt + && TARGET_CLASSES.contains(&class.name.as_str()) + { + widen_class(class, &mut edits); + } + } + + // a widened method references `Never` in its default; make sure the name + // resolves. done last so it is skipped when nothing was widened + if !edits.is_empty() + && let Some(import) = ensure_never_import(body) + { + edits.push(import); + } + edits + } +} + +/// widen every fresh-container-returning method of one container class +fn widen_class(class: &StmtClassDef, edits: &mut Vec) { + let Some(type_params) = &class.type_params else { + return; + }; + // the class's type parameters in order, paired with whether each is invariant + let params: Vec = type_params + .iter() + .map(|tp| ClassParam { + name: tp.name().as_str(), + invariant: matches!(tp, TypeParam::TypeVar(t) if t.variance == Some(Variance::Invariant)), + }) + .collect(); + + let mut methods = Vec::new(); + collect_methods(&class.body, &mut methods); + for func in methods { + widen_method(func, class.name.as_str(), ¶ms, edits); + } +} + +/// collect the class's methods, descending through `if`/`else` version guards +/// but not into nested classes or a method's own body — those own their scopes +fn collect_methods<'a>(body: &'a [Stmt], out: &mut Vec<&'a StmtFunctionDef>) { + for stmt in body { + match stmt { + Stmt::FunctionDef(func) => out.push(func), + Stmt::If(if_stmt) => { + collect_methods(&if_stmt.body, out); + for clause in &if_stmt.elif_else_clauses { + collect_methods(&clause.body, out); + } + } + _ => {} + } + } +} + +struct ClassParam<'a> { + name: &'a str, + invariant: bool, +} + +/// widen a single method if it returns a fresh instance of its own class, i.e. +/// `ClassName[...]` with one subscript position per class type parameter +fn widen_method( + func: &StmtFunctionDef, + class_name: &str, + params: &[ClassParam], + edits: &mut Vec, +) { + // only widen methods reached by an ordinary call (`copy`, `union`, + // `difference`, ...), never operator dunders (`__add__`, `__getitem__`, ...). + // a call already threads the caller's expected type into inference, so the + // `Never`-defaulted parameter solves for free; an operator's expected type is + // not, and making it so requires bidirectional inference on every binary op + // and subscript — far too expensive on real code (it times out projects like + // xarray). on an operator the parameter would just be dead weight + if is_dunder(func.name.as_str()) { + return; + } + + // idempotent: a previous run already added the widening parameter(s) + if let Some(type_params) = &func.type_params + && type_params + .iter() + .any(|p| p.name().as_str().starts_with(WIDEN_PREFIX)) + { + return; + } + + let Some(Expr::Subscript(sub)) = func.returns.as_deref() else { + return; + }; + let Expr::Name(head) = sub.value.as_ref() else { + return; + }; + if head.id.as_str() != class_name { + return; + } + + let positions: Vec<&Expr> = match sub.slice.as_ref() { + Expr::Tuple(tuple) => tuple.elts.iter().collect(), + single => vec![single], + }; + // only a return that mirrors the class's own arity is a fresh same-class + // container; anything else (e.g. `dict.keys() -> KeysView[Key]`) is skipped + // by the head check above, and defensive arity keeps position/param aligned + if positions.len() != params.len() { + return; + } + + let mut used: HashSet = params.iter().map(|p| p.name.to_string()).collect(); + if let Some(type_params) = &func.type_params { + used.extend(type_params.iter().map(|p| p.name().to_string())); + } + + let mut new_params: Vec = Vec::new(); + let mut position_inserts: Vec = Vec::new(); + for (position, param) in positions.iter().zip(params) { + // widen only invariant positions that actually carry the class parameter + if !param.invariant || !references(position, param.name) { + continue; + } + let widen = unique_name(&format!("{WIDEN_PREFIX}{}", param.name), &mut used); + let at = position.range().end().to_usize(); + position_inserts.push(Edit { + start: at, + end: at, + replacement: format!(" | {widen}"), + }); + new_params.push(format!("{widen} = Never")); + } + + if new_params.is_empty() { + return; + } + edits.extend(position_inserts); + edits.push(type_param_edit(func, &new_params)); +} + +/// prefix identifying the synthesized widening parameters (drives idempotency) +const WIDEN_PREFIX: &str = "Widen"; + +/// insert the widening parameters into the method's type-parameter list, +/// creating the list when the method has none +fn type_param_edit(func: &StmtFunctionDef, new_params: &[String]) -> Edit { + match &func.type_params { + // append before the closing `]`; existing params carry no default, so + // the defaulted widening params correctly sort last + Some(type_params) => { + let close = type_params.range().end().to_usize() - 1; + Edit { + start: close, + end: close, + replacement: format!(", {}", new_params.join(", ")), + } + } + // fresh `[...]` right after the method name + None => { + let at = func.name.range().end().to_usize(); + Edit { + start: at, + end: at, + replacement: format!("[{}]", new_params.join(", ")), + } + } + } +} + +/// add `Never` to the module's `typing_extensions` import, or `None` if it is +/// already imported. falls back to a fresh import line if the module has no +/// `typing_extensions` import to extend +fn ensure_never_import(body: &[Stmt]) -> Option { + let mut fallback_anchor = None; + for stmt in body { + if let Stmt::ImportFrom(import) = stmt + && import + .module + .as_ref() + .is_some_and(|m| m == NEVER_IMPORT_FROM) + { + if import + .names + .iter() + .any(|alias| alias.name.as_str() == "Never") + { + return None; + } + let first = import.names.first()?; + let at = first.name.range().start().to_usize(); + return Some(Edit { + start: at, + end: at, + replacement: "Never, ".to_string(), + }); + } + if fallback_anchor.is_none() && matches!(stmt, Stmt::ImportFrom(_) | Stmt::Import(_)) { + fallback_anchor = Some(stmt.range().start().to_usize()); + } + } + fallback_anchor.map(|at| Edit { + start: at, + end: at, + replacement: format!("from {NEVER_IMPORT_FROM} import Never\n"), + }) +} + +/// whether `name` appears as a bare name anywhere in `expr` +fn references(expr: &Expr, name: &str) -> bool { + let mut finder = NameFinder { name, found: false }; + finder.visit_expr(expr); + finder.found +} + +struct NameFinder<'a> { + name: &'a str, + found: bool, +} + +impl<'a> SourceOrderVisitor<'a> for NameFinder<'a> { + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Name(name) = expr + && name.id.as_str() == self.name + { + self.found = true; + } + if !self.found { + walk_expr(self, expr); + } + } +} + +/// whether `name` is a dunder such as `__add__` or `__getitem__` +fn is_dunder(name: &str) -> bool { + name.len() > 4 && name.starts_with("__") && name.ends_with("__") +} + +/// `candidate`, suffixed with the smallest integer that avoids a collision with +/// `used`; records the chosen name in `used` +fn unique_name(candidate: &str, used: &mut HashSet) -> String { + let mut chosen = candidate.to_string(); + let mut suffix = 2; + while used.contains(&chosen) { + chosen = format!("{candidate}{suffix}"); + suffix += 1; + } + used.insert(chosen.clone()); + chosen +} + +#[cfg(test)] +mod tests { + use super::*; + use ruff_python_ast::PySourceType; + use ruff_python_parser::parse_unchecked_source; + + use crate::apply_edits; + + fn run(path: &str, src: &str) -> String { + let parsed = parse_unchecked_source(src, PySourceType::BasedPythonStub); + let edits = OutputWidening.rewrite(Path::new(path), &parsed, src); + apply_edits(src, edits) + } + + #[test] + fn widens_pure_output_copy() { + let src = "\ +from typing_extensions import Self +class list[in out Element]: + def copy(self) -> list[Element]: ... + def append(self, object: Element, /) -> None: ... +"; + let expected = "\ +from typing_extensions import Never, Self +class list[in out Element]: + def copy[WidenElement = Never](self) -> list[Element | WidenElement]: ... + def append(self, object: Element, /) -> None: ... +"; + assert_eq!(run("builtins.byi", src), expected); + } + + #[test] + fn extends_existing_type_params() { + // `union` already carries a method type parameter inferred from its argument; the widening + // parameter is appended after it + let src = "\ +from typing_extensions import Never +class set[in out Element]: + def union[Other](self, *s: Iterable[Other]) -> set[Element | Other]: ... +"; + let expected = "\ +from typing_extensions import Never +class set[in out Element]: + def union[Other, WidenElement = Never](self, *s: Iterable[Other]) -> set[Element | Other | WidenElement]: ... +"; + assert_eq!(run("builtins.byi", src), expected); + } + + #[test] + fn widens_each_invariant_position_of_a_multi_param_class() { + let src = "\ +from typing_extensions import Never +class dict[in out Key, in out Value]: + def copy(self) -> dict[Key, Value]: ... +"; + let expected = "\ +from typing_extensions import Never +class dict[in out Key, in out Value]: + def copy[WidenKey = Never, WidenValue = Never](self) -> dict[Key | WidenKey, Value | WidenValue]: ... +"; + assert_eq!(run("builtins.byi", src), expected); + } + + #[test] + fn leaves_operator_dunders_unwidened() { + // operator dunders can only widen through bidirectional inference on the operator, which is + // deliberately not attempted, so they are left alone even though they return a fresh `list` + let src = "\ +class list[in out Element]: + def __add__[Other](self, value: list[Other], /) -> list[Other | Element]: ... + def __mul__(self, value: SupportsIndex, /) -> list[Element]: ... + def __getitem__(self, s: slice[SupportsIndex | None], /) -> list[Element]: ... +"; + assert_eq!(run("builtins.byi", src), src); + } + + #[test] + fn leaves_covariant_containers_untouched() { + let src = "\ +class frozenset[out Element]: + def copy(self) -> frozenset[Element]: ... +"; + assert_eq!(run("builtins.byi", src), src); + } + + #[test] + fn ignores_returns_that_are_not_a_fresh_same_class_container() { + let src = "\ +class list[in out Element]: + def pop(self, index: SupportsIndex = -1, /) -> Element: ... + def __iter__(self) -> Iterator[Element]: ... + def __iadd__(self, value: Iterable[Element], /) -> Self: ... + def clear(self) -> None: ... +"; + assert_eq!(run("builtins.byi", src), src); + } + + #[test] + fn descends_into_version_guards_but_not_nested_classes() { + let src = "\ +from typing_extensions import Never +class dict[in out Key, in out Value]: + if sys.version_info >= (3, 9): + def merged(self, other: dict[Key, Value], /) -> dict[Key, Value]: ... + class _NestedView[in out Element]: + def copy(self) -> _NestedView[Element]: ... +"; + let expected = "\ +from typing_extensions import Never +class dict[in out Key, in out Value]: + if sys.version_info >= (3, 9): + def merged[WidenKey = Never, WidenValue = Never](self, other: dict[Key, Value], /) -> dict[Key | WidenKey, Value | WidenValue]: ... + class _NestedView[in out Element]: + def copy(self) -> _NestedView[Element]: ... +"; + assert_eq!(run("builtins.byi", src), expected); + } + + #[test] + fn idempotent_when_already_widened() { + let src = "\ +from typing_extensions import Never +class list[in out Element]: + def copy[WidenElement = Never](self) -> list[Element | WidenElement]: ... +"; + assert_eq!(run("builtins.byi", src), src); + } + + #[test] + fn skips_non_builtins_modules() { + let src = "\ +class list[in out Element]: + def copy(self) -> list[Element]: ... +"; + assert_eq!(run("collections.byi", src), src); + } + + #[test] + fn skips_untargeted_classes() { + let src = "\ +class MyList[in out Element]: + def copy(self) -> MyList[Element]: ... +"; + assert_eq!(run("builtins.byi", src), src); + } +} diff --git a/crates/ty_ide/src/type_hierarchy.rs b/crates/ty_ide/src/type_hierarchy.rs index e7b75d9a18..ba0222d6da 100644 --- a/crates/ty_ide/src/type_hierarchy.rs +++ b/crates/ty_ide/src/type_hierarchy.rs @@ -210,7 +210,7 @@ mod tests { let supertypes = test.supertypes(); insta::assert_snapshot!( snapshot(&test.db, &supertypes), - @"vendored://stdlib/builtins.byi:2677:2683 object :: builtins", + @"vendored://stdlib/builtins.byi:2684:2690 object :: builtins", ); } @@ -424,12 +424,12 @@ mod tests { let item = test.prepare().unwrap(); insta::assert_snapshot!( snapshot(&test.db, &[item]), - @"vendored://stdlib/builtins.byi:7359:7363 type :: builtins", + @"vendored://stdlib/builtins.byi:7366:7370 type :: builtins", ); let supertypes = test.supertypes(); insta::assert_snapshot!( snapshot(&test.db, &supertypes), - @"vendored://stdlib/builtins.byi:2677:2683 object :: builtins", + @"vendored://stdlib/builtins.byi:2684:2690 object :: builtins", ); } @@ -481,7 +481,7 @@ mod tests { let supertypes = test.supertypes(); insta::assert_snapshot!( snapshot(&test.db, &supertypes), - @"vendored://stdlib/builtins.byi:100771:100776 tuple :: builtins", + @"vendored://stdlib/builtins.byi:100778:100783 tuple :: builtins", ); } diff --git a/crates/ty_python_semantic/resources/mdtest/generics/output_widening.md b/crates/ty_python_semantic/resources/mdtest/generics/output_widening.md new file mode 100644 index 0000000000..e52ca7202b --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/generics/output_widening.md @@ -0,0 +1,128 @@ +# Output-position widening for invariant containers + +An invariant generic container cannot be assigned to a wider specialization — `list[int]` is not a +`list[int | None]`, since a holder of the wider type could insert a `None` the original never +expected. But a method that returns a *fresh* container of the same class (`copy`, the set algebra, +...) hands back a brand new object the caller solely owns, so widening its element type at the call +site is sound. + +basedpython encodes this in the typeshed stubs (see the `output-widening` patch): each such method +gains a `Never`-defaulted type parameter unioned into every invariant position of its return. With +an expected type the parameter solves to the widening; with no expected type it defaults to `Never` +and `T | Never` collapses back to `T`, so ordinary inference is unchanged. + +Only methods reached by an ordinary call are widened — a call already threads the caller's expected +type into inference. Operator dunders (`+`, `*`, `[]`, `&`, `|`, ...) are left alone: making their +returns widen would require bidirectional inference on every binary op and subscript, which is far +too expensive on real code. + +```toml +[environment] +python-version = "3.13" +``` + +## `list` + +```py +def f(a: list[int]): + widened: list[int | None] = a.copy() + reveal_type(a.copy()) # revealed: list[int] + + # widening only ever *adds* to the union; it cannot replace the element type + # error: [invalid-assignment] + bad: list[str] = a.copy() +``` + +## `set` + +Every fresh-set method widens, including the ones that already infer an element from their argument: + +```py +def f(s: set[int]): + via_copy: set[int | None] = s.copy() + via_difference: set[int | None] = s.difference({1}) + via_intersection: set[int | None] = s.intersection({1}) + via_union: set[int | str | None] = s.union(["x"]) + via_symmetric: set[int | str | None] = s.symmetric_difference(["x"]) + + reveal_type(s.copy()) # revealed: set[int] + reveal_type(s.union(["x"])) # revealed: set[int | str] +``` + +## `dict` + +Each invariant position is widened independently: + +```py +def f(m: dict[str, int]): + both: dict[str | None, int | None] = m.copy() + key_only: dict[str | None, int] = m.copy() + value_only: dict[str, int | None] = m.copy() + + reveal_type(m.copy()) # revealed: dict[str, int] +``` + +## Covariant containers are unaffected + +`frozenset` is immutable and therefore covariant, so its copies already widen without any special +machinery — and the patch correctly leaves it alone: + +```py +def f(s: frozenset[int]): + widened: frozenset[int | None] = s.copy() + reveal_type(s.copy()) # revealed: frozenset[int] +``` + +## Operators are not widened + +The result of an operator keeps its natural type; assigning it to a wider invariant specialization +is still rejected (this behaves exactly as it did before the feature): + +```py +def f(a: list[int]): + # error: [invalid-assignment] + bad: list[int | None] = a + a +``` + +Heterogeneous operators keep resolving to their natural result, unchanged: + +```py +class A: ... +class B: ... + +def g(x: list[A], y: list[B]): + z: list[A | B] = x + y + reveal_type(x + y) # revealed: list[B | A] +``` + +## User-defined invariant classes + +The same pattern applies to any invariant class. `Widen` defaults to `Never`, so `T | Widen` +collapses to `T` when the caller gives no expected type. (The return annotation is quoted because +basedpython evaluates annotations eagerly, and the class is not yet bound inside its own body.) + +```py +from typing import Never + +class Box[T]: + def push(self, value: T) -> None: ... + def copy[Widen = Never](self) -> "Box[T | Widen]": + raise NotImplementedError + +def f(b: Box[int]): + widened: Box[int | None] = b.copy() + reveal_type(b.copy()) # revealed: Box[int] +``` + +Without the widening parameter, an invariant box cannot be widened at all: + +```py +class Plain[T]: + def push(self, value: T) -> None: ... + def copy(self) -> "Plain[T]": + raise NotImplementedError + +def f(p: Plain[int]): + # error: [invalid-assignment] + widened: Plain[int | None] = p.copy() +``` diff --git a/crates/ty_vendored/vendor/typeshed/stdlib/builtins.byi b/crates/ty_vendored/vendor/typeshed/stdlib/builtins.byi index 3ec5d0dc55..85acf966b0 100644 --- a/crates/ty_vendored/vendor/typeshed/stdlib/builtins.byi +++ b/crates/ty_vendored/vendor/typeshed/stdlib/builtins.byi @@ -52,7 +52,7 @@ from types import CellType, CodeType, EllipsisType, GenericAlias, NotImplemented from typing import IO, Any, BinaryIO, ClassVar, Concatenate, Final, Generic, Mapping, MutableMapping, MutableSequence, ParamSpec, Protocol, Sequence, SupportsAbs, SupportsBytes, SupportsComplex, SupportsFloat, SupportsIndex, TypeAlias, TypeGuard, TypeVar, final, type_check_only # we can't import `Literal` from typing or mypy crashes: see #11247 -from typing_extensions import Literal, LiteralString, Self, TypeIs, TypeVarTuple, deprecated, disjoint_base # noqa: Y023, UP035 +from typing_extensions import Never, Literal, LiteralString, Self, TypeIs, TypeVarTuple, deprecated, disjoint_base # noqa: Y023, UP035 if sys.version_info >= (3, 14): from _typeshed import AnnotateFunc @@ -2769,7 +2769,7 @@ class list[in out Element](MutableSequence[Element]): def __init__(self) -> None def __init__(self, iterable: Iterable[Element], /) -> None - def copy(self) -> list[Element]: + def copy[WidenElement = Never](self) -> list[Element | WidenElement]: """Return a shallow copy of the list.""" def append(self, object: Element, /) -> None: @@ -2909,7 +2909,7 @@ class dict[in out Key, in out Value](MutableMapping[Key, Value]): def __init__(self: dict[bytes, bytes], iterable: Iterable[list[bytes]], /) -> None def __new__(cls, /, *args: dynamic, **kwargs: dynamic) -> Self - def copy(self) -> dict[Key, Value]: + def copy[WidenKey = Never, WidenValue = Never](self) -> dict[Key | WidenKey, Value | WidenValue]: """Return a shallow copy of the dict.""" def keys(self) -> dict_keys[Key, Value]: @@ -3071,10 +3071,10 @@ class set[in out Element](MutableSet[Element]): This has no effect if the element is already present. """ - def copy(self) -> set[Element]: + def copy[WidenElement = Never](self) -> set[Element | WidenElement]: """Return a shallow copy of a set.""" - def difference(self, *s: Iterable[object]) -> set[Element]: + def difference[WidenElement = Never](self, *s: Iterable[object]) -> set[Element | WidenElement]: """Return a new set with elements in the set that are not in the others.""" def difference_update(self, *s: Iterable[object]) -> None: @@ -3087,7 +3087,7 @@ class set[in out Element](MutableSet[Element]): an exception when an element is missing from the set. """ - def intersection(self, *s: Iterable[object]) -> set[Element]: + def intersection[WidenElement = Never](self, *s: Iterable[object]) -> set[Element | WidenElement]: """Return a new set with elements common to the set and all others.""" def intersection_update(self, *s: Iterable[object]) -> None: @@ -3108,13 +3108,13 @@ class set[in out Element](MutableSet[Element]): If the element is not a member, raise a KeyError. """ - def symmetric_difference[Other](self, s: Iterable[Other], /) -> set[Element | Other]: + def symmetric_difference[Other, WidenElement = Never](self, s: Iterable[Other], /) -> set[Element | Other | WidenElement]: """Return a new set with elements in either the set or other but not both.""" def symmetric_difference_update(self, s: Iterable[Element], /) -> None: """Update the set, keeping only elements found in either set, but not in both.""" - def union[Other](self, *s: Iterable[Other]) -> set[Element | Other]: + def union[Other, WidenElement = Never](self, *s: Iterable[Other]) -> set[Element | Other | WidenElement]: """Return a new set with elements from the set and all others.""" def update(self, *s: Iterable[Element]) -> None: