Django template language services

_typos.toml

@@ -25,6 +25,7 @@ BA = "BA" # acronym for "Bad Allowed", used in testing.
 jod = "jod" # e.g., `jod-thread`
 Numer = "Numer" # Library name 'NumerBlox' in "Who's Using Ruff?"
 CPY = "CPY"  # it's a Ruff rule category
+aadd = "aadd"  # django's async `add`, alongside `acreate`, `aset` and the rest
 
 [default]
 extend-ignore-re = [

crates/by_transforms/Cargo.toml

@@ -21,6 +21,7 @@ ruff_python_stdlib = { workspace = true }
 ruff_python_trivia = { workspace = true }
 ruff_source_file = { workspace = true }
 ruff_text_size = { workspace = true }
+salsa = { workspace = true }
 thin-vec = { workspace = true }
 ty_project = { workspace = true, features = ["testing"] }
 ty_python_core = { workspace = true }

crates/by_transforms/src/transforms/ast_driver.rs

@@ -36,16 +36,16 @@ use ruff_text_size::{Ranged, TextRange};
 use super::{
     annotation, anon_named_tuple, auto_quote, callable, character_type, checked_cast, coalesce,
     coalesce_chain, compat, context_params, conversion, decl_site_variance, decorator_keyword,
-    dedent_string, destructure, dynamic_keyword, empty_declarations, export_import, extension,
-    float_const, force_unwrap, frameworks, generic_call, generics, grapheme_string, identity_swap,
-    if_let, implementation, implicit_receiver, implicit_typing, inferred_annotation, init_method,
-    just_float, kw_subscript, literal_string, literal_types, local_once, main_function, match_type,
-    modifiers, mutable_defaults, none_chain, optional_type, overload, parametric_is, postfix_await,
-    propagate, properties, protocol_type, raises_clause, reified_generic, repeated_underscore,
-    sentinel, some_ctor, soundness, statement_expression, string_tag, super_keyword,
-    symbolic_type_op, top_star, trailing_lambda, tuple_index, type_fn, type_is, type_reification,
-    typed_dict_literal, typed_lambda, typeof_keyword, unique_loop_bindings, unpack,
-    use_site_variance,
+    dedent_string, destructure, django_lookup, dynamic_keyword, empty_declarations, export_import,
+    extension, float_const, force_unwrap, frameworks, generic_call, generics, grapheme_string,
+    identity_swap, if_let, implementation, implicit_receiver, implicit_typing, inferred_annotation,
+    init_method, just_float, kw_subscript, literal_string, literal_types, local_once,
+    main_function, match_type, modifiers, mutable_defaults, none_chain, optional_type, overload,
+    parametric_is, postfix_await, propagate, properties, protocol_type, raises_clause,
+    reified_generic, repeated_underscore, sentinel, some_ctor, soundness, statement_expression,
+    string_tag, super_keyword, symbolic_type_op, top_star, trailing_lambda, tuple_index, type_fn,
+    type_is, type_reification, typed_dict_literal, typed_lambda, typeof_keyword,
+    unique_loop_bindings, unpack, use_site_variance,
 };
 use crate::Config;
 use crate::type_info::TypeInfo;
@@ -555,6 +555,7 @@ pub(crate) fn run_against_source<'a>(
     let implementation_block_pass = implementation::ImplementationBlockPass::new(source_ref);
     let conversion_pass = conversion::ConversionPass::new(source_ref);
     let implicit_receiver_pass = implicit_receiver::ImplicitReceiverPass;
+    let django_lookup_pass = django_lookup::DjangoLookupPass;
     let frameworks_pass = frameworks::FrameworksPass::new(source_ref);
     let variance_pass = decl_site_variance::VarianceStripPass::new(source_ref);
     let anon_named_tuple_pass =
@@ -698,6 +699,11 @@ pub(crate) fn run_against_source<'a>(
         // members → its receiver parameter. same shape as the extension rewrite
         // above, which wins when both could apply
         &implicit_receiver_pass,
+        // django lookups written as expressions (`filter(author.name == "x")`)
+        // become keyword arguments. the argument is replaced whole, with the
+        // value passing through as `Src`, so lowerings inside it still compose;
+        // disjoint from the call rewrites above, which target the callee
+        &django_lookup_pass,
         &dynamic_keyword_pass,
         // import-only companion to the ty-side implicit `Character` resolution;
         // emits no text edits, so ordering among the type passes is free

crates/by_transforms/src/transforms/context_sensitive.rs

@@ -86,7 +86,7 @@ mod tests {
     fn mapped_by_line(source: &str, needle: &str) -> Option<u32> {
         let (db, file) = make_in_memory_db(source);
         let (output, line_map) =
-            transpile_typed_with_map(&db, file, &Config::test_default()).unwrap();
+            transpile_typed_with_map(&db, file, &Config::test_default(), None).unwrap();
         let index = output
             .lines()
             .position(|line| line.trim() == needle)

crates/by_transforms/src/transforms/django_lookup.rs

@@ -0,0 +1,83 @@
+//! Lowering for django lookups written as expressions.
+//!
+//! `Book.objects.filter(author.name == "Ursula", published > date(1970, 1, 1))`
+//! lowers to `Book.objects.filter(author__name="Ursula",
+//! published__gt=date(1970, 1, 1))`.
+//!
+//! Which arguments are lookups is ty's answer, not this pass's: the names in a
+//! lookup path resolve to model fields rather than to anything in scope, and the
+//! checker and the lowering read that from one query, so the query the file was
+//! checked against is the query it lowers to. Every argument ty did not read as
+//! a lookup passes through untouched.
+//!
+//! The value is re-emitted as a source span, so lowerings inside it still apply.
+
+use ruff_python_ast::visitor::{Visitor, walk_expr};
+use ruff_python_ast::{Expr, Stmt};
+
+use super::ast_driver::{Fragment, PassContext, TypeAwarePass};
+use crate::type_info::TypeInfo;
+
+struct DjangoLookupLower<'a> {
+    types: &'a dyn TypeInfo,
+    edits: Vec<(ruff_text_size::TextRange, Vec<Fragment>)>,
+}
+
+impl<'ast> Visitor<'ast> for DjangoLookupLower<'_> {
+    fn visit_expr(&mut self, expr: &'ast Expr) {
+        if let Expr::Call(call) = expr {
+            for lookup in self.types.django_lookup_arguments(call) {
+                self.edits.push((
+                    lookup.argument,
+                    vec![
+                        Fragment::Lit(format!("{}=", lookup.key)),
+                        Fragment::Src(lookup.value),
+                    ],
+                ));
+            }
+        }
+        walk_expr(self, expr);
+    }
+}
+
+pub(crate) struct DjangoLookupPass;
+
+impl TypeAwarePass for DjangoLookupPass {
+    fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) {
+        let mut inner = DjangoLookupLower {
+            types,
+            edits: Vec::new(),
+        };
+        for stmt in stmts {
+            inner.visit_stmt(stmt);
+        }
+        ctx.template_edits.extend(inner.edits);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use indoc::indoc;
+
+    use crate::{Config, transpile};
+
+    /// A file with no django in it must come out untouched. The lowering is
+    /// keyed entirely on ty resolving the callee to a lookup method on a model,
+    /// so a comparison argument to any other call is an ordinary argument —
+    /// which is what makes the rewrite additive rather than a rule about the
+    /// word `filter`.
+    #[test]
+    fn a_comparison_argument_to_an_ordinary_call_is_untouched() {
+        for source in [
+            "def filter(*args): ...\nfilter(1 == 2)\n",
+            "class M:\n    def filter(self, *args): ...\n\nM().filter(1 == 2)\n",
+            indoc! {"
+                def use(objects) -> None:
+                    objects.filter(objects == 1)
+            "},
+        ] {
+            let out = transpile(source, &Config::test_default()).expect("transpile should succeed");
+            assert!(out.contains("== "), "for `{source}` got:\n{out}");
+        }
+    }
+}

crates/by_transforms/src/transforms/frameworks.rs

@@ -137,7 +137,7 @@ mod tests {
 
     fn transpile_result(db: &TestDb, path: &str) -> Result<String, String> {
         let file = system_path_to_file(db, path).expect("file not in db");
-        transpile_typed(db, file, &Config::test_default()).map_err(|err| err.to_string())
+        transpile_typed(db, file, &Config::test_default(), None).map_err(|err| err.to_string())
     }
 
     /// a project db with a mock django package in site-packages (`Model` must
@@ -248,7 +248,7 @@ mod tests {
             soundness: crate::config::SoundnessPositions::all(),
             ..Config::default()
         };
-        let out = transpile_typed(&db, file, &config).expect("model should transpile");
+        let out = transpile_typed(&db, file, &config, None).expect("model should transpile");
         assert!(
             out.contains("class User(DeclarativeBase):"),
             "class structure should survive, got:\n{out}"
@@ -316,7 +316,7 @@ mod tests {
             soundness: crate::config::SoundnessPositions::all(),
             ..Config::default()
         };
-        let out = transpile_typed(&db, file, &config).expect("model should transpile");
+        let out = transpile_typed(&db, file, &config, None).expect("model should transpile");
         assert!(
             out.contains("class Author(models.Model):"),
             "class structure should survive, got:\n{out}"
@@ -424,7 +424,8 @@ mod tests {
             min_version: ruff_python_ast::PythonVersion::PY313,
             ..Config::test_default()
         };
-        let out = transpile_typed(&db, file, &config).expect("generic model should transpile");
+        let out =
+            transpile_typed(&db, file, &config, None).expect("generic model should transpile");
         assert!(
             !out.contains("@generic"),
             "a model class must never be wrapped with `@generic`, got:\n{out}"
@@ -462,7 +463,7 @@ mod tests {
             soundness: crate::config::SoundnessPositions::all(),
             ..Config::default()
         };
-        let out = transpile_typed(&db, file, &config).expect("model should transpile");
+        let out = transpile_typed(&db, file, &config, None).expect("model should transpile");
         assert!(
             out.contains("class User(BaseModel):"),
             "class structure should survive, got:\n{out}"

crates/by_transforms/src/transforms/kw_subscript.rs

@@ -393,7 +393,7 @@ mod tests {
                 .expect("write file failed");
             db.init_program().expect("program init failed");
             let file = system_path_to_file(&db, "/proj/main.by").expect("file not in db");
-            transpile_typed(&db, file, &Config::test_default()).expect("transpile failed")
+            transpile_typed(&db, file, &Config::test_default(), None).expect("transpile failed")
         }
 
         #[test]

crates/by_transforms/src/transforms/mod.rs

@@ -15,6 +15,7 @@ pub(crate) mod decl_site_variance;
 pub(crate) mod decorator_keyword;
 pub(crate) mod dedent_string;
 pub(crate) mod destructure;
+pub(crate) mod django_lookup;
 pub(crate) mod dynamic_keyword;
 pub(crate) mod empty_declarations;
 pub(crate) mod enums;

crates/by_transforms/src/type_info.rs

@@ -181,6 +181,16 @@ pub(crate) trait TypeInfo {
     /// it used unqualified. both lower to the block's receiver parameter
     fn implicit_receiver_name(&self, name: &ExprName) -> Option<ImplicitReceiverReference>;
 
+    /// how each positional argument of a django lookup method that spells a
+    /// `__` lookup as an expression lowers to a keyword —
+    /// `filter(author.name == "x")` → `filter(author__name="x")`. empty for
+    /// every other call, and for any argument the checker did not read as a
+    /// lookup, which must then be left exactly as written
+    fn django_lookup_arguments(
+        &self,
+        call: &ruff_python_ast::ExprCall,
+    ) -> Vec<ty_python_semantic::DjangoLookupArgument>;
+
     /// the enum a *context-sensitively* resolved name must be qualified with —
     /// `Red` in a `Color` context lowers to `Color.Red`. `None` for every name
     /// that resolves the ordinary way
@@ -587,6 +597,13 @@ impl TypeInfo for SemanticModel<'_> {
         SemanticModel::implicit_receiver_name(self, name)
     }
 
+    fn django_lookup_arguments(
+        &self,
+        call: &ruff_python_ast::ExprCall,
+    ) -> Vec<ty_python_semantic::DjangoLookupArgument> {
+        SemanticModel::django_lookup_arguments(self, call)
+    }
+
     fn context_sensitive_qualifier(&self, name: &ExprName) -> Option<String> {
         SemanticModel::context_sensitive_qualifier(self, name)
     }

crates/ruff_linter/resources/test/fixtures/pyflakes/F821_basedpython.by

@@ -192,3 +192,25 @@ print(ImplementedAsInterface(Implemented()))
 # check` reports it as an unresolved type
 implementation UndefinedInterface for UndefinedImplemented:  # F821
     pass
+
+
+# a django lookup expression: the transpiler lowers the field path to the
+# keyword it spells (`filter(author.name == "x")` -> `filter(author__name="x")`),
+# so the path is never evaluated and none of these names is undefined
+class Model:
+    objects: object
+
+
+Book = Model()
+Book.objects.filter(author.name == "Ursula")
+Book.objects.filter(published > 1)
+Book.objects.filter(data["key"] == 1)
+Book.objects.exclude(title == "x", author.pk >= 3)
+Book.objects.filter(pk in [1, 2, 3])
+Book.objects.get(pk == 1)
+
+# but the suppression is only for that shape, so each of these is still reported
+Book.objects.filter(bare_name)  # F821
+Book.objects.annotate(not_a_lookup_method.x == 1)  # F821
+Book.objects.filter(data[undefined_index] == 1)  # F821
+Book.objects.filter(no_lookup_for_ne != 1)  # F821

crates/ruff_linter/src/rules/pyflakes/snapshots/ruff_linter__rules__pyflakes__tests__F821_F821_basedpython.by.snap

@@ -176,3 +176,43 @@ F821 Undefined name `UndefinedInterface`
     |                ^^^^^^^^^^^^^^^^^^
 194 |     pass
     |
+
+F821 Undefined name `bare_name`
+   --> F821_basedpython.by:213:21
+    |
+212 | # but the suppression is only for that shape, so each of these is still reported
+213 | Book.objects.filter(bare_name)  # F821
+    |                     ^^^^^^^^^
+214 | Book.objects.annotate(not_a_lookup_method.x == 1)  # F821
+215 | Book.objects.filter(data[undefined_index] == 1)  # F821
+    |
+
+F821 Undefined name `not_a_lookup_method`
+   --> F821_basedpython.by:214:23
+    |
+212 | # but the suppression is only for that shape, so each of these is still reported
+213 | Book.objects.filter(bare_name)  # F821
+214 | Book.objects.annotate(not_a_lookup_method.x == 1)  # F821
+    |                       ^^^^^^^^^^^^^^^^^^^
+215 | Book.objects.filter(data[undefined_index] == 1)  # F821
+216 | Book.objects.filter(no_lookup_for_ne != 1)  # F821
+    |
+
+F821 Undefined name `undefined_index`
+   --> F821_basedpython.by:215:26
+    |
+213 | Book.objects.filter(bare_name)  # F821
+214 | Book.objects.annotate(not_a_lookup_method.x == 1)  # F821
+215 | Book.objects.filter(data[undefined_index] == 1)  # F821
+    |                          ^^^^^^^^^^^^^^^
+216 | Book.objects.filter(no_lookup_for_ne != 1)  # F821
+    |
+
+F821 Undefined name `no_lookup_for_ne`
+   --> F821_basedpython.by:216:21
+    |
+214 | Book.objects.annotate(not_a_lookup_method.x == 1)  # F821
+215 | Book.objects.filter(data[undefined_index] == 1)  # F821
+216 | Book.objects.filter(no_lookup_for_ne != 1)  # F821
+    |                     ^^^^^^^^^^^^^^^^
+    |