From fcb400328d6584948d4099b603919b837da8f0e5 Mon Sep 17 00:00:00 2001 From: charliecloudberry Date: Thu, 6 Aug 2026 02:27:36 +0200 Subject: [PATCH 01/16] support django template files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit templates get completions, go-to-definition and semantic highlighting, all of it joined to the project's own django definitions rather than to a grammar: `{{ book.author. }}` completes model fields because some view wrote `render(request, "post.html", {"book": book})` and the type checker knows what that `book` is. the template front end is new — a lexer for the three constructs, and an index that recovers block nesting from the tags that impose it, since the delimiters do not. everything else is a join to the python side: the `templatetags` modules a `{% load %}` names, the `path(…, name=…)` a `{% url %}` reverses, the `render()` call a `{{ variable }}` comes from. the server recognises a template from the editor's language id, or from the path when the editor just says `html`. a template's `File` is interned so its source can be read, but it never enters the project's open-file set, so the type checker is never handed an html file. the semantic token legend grows two types and a modifier. both are appended: a token type's position in the enum is the index the wire format sends, so inserting would silently recolour every python file. --- crates/ty_ide/src/django_template.rs | 232 ++++ crates/ty_ide/src/django_template/builtins.rs | 769 ++++++++++++ .../ty_ide/src/django_template/completion.rs | 1084 +++++++++++++++++ crates/ty_ide/src/django_template/goto.rs | 492 ++++++++ crates/ty_ide/src/django_template/index.rs | 1016 +++++++++++++++ crates/ty_ide/src/django_template/lexer.rs | 917 ++++++++++++++ crates/ty_ide/src/django_template/project.rs | 740 +++++++++++ crates/ty_ide/src/django_template/resolve.rs | 222 ++++ .../src/django_template/semantic_tokens.rs | 357 ++++++ crates/ty_ide/src/lib.rs | 8 +- crates/ty_ide/src/semantic_tokens.rs | 28 +- .../src/types/ide_support.rs | 10 + crates/ty_server/src/capabilities.rs | 16 +- .../ty_server/src/document/text_document.rs | 36 +- .../ty_server/src/server/api/diagnostics.rs | 8 + .../src/server/api/notifications/did_open.rs | 2 +- .../api/notifications/did_open_notebook.rs | 2 +- .../src/server/api/requests/completion.rs | 90 ++ .../server/api/requests/goto_definition.rs | 10 +- .../server/api/requests/semantic_tokens.rs | 4 +- .../api/requests/semantic_tokens_range.rs | 1 + .../src/server/api/semantic_tokens.rs | 14 +- crates/ty_server/src/session.rs | 47 +- crates/ty_server/src/session/index.rs | 2 +- crates/ty_server/src/system.rs | 4 +- .../ty_server/tests/e2e/django_templates.rs | 259 ++++ crates/ty_server/tests/e2e/main.rs | 14 +- .../e2e__initialize__initialization.snap | 12 +- ...ialize__initialization_with_workspace.snap | 12 +- crates/ty_wasm/src/lib.rs | 4 + .../frameworks/django-templates.md | 118 ++ docs/basedpython/frameworks/django.md | 5 + zensical.toml | 1 + 33 files changed, 6501 insertions(+), 35 deletions(-) create mode 100644 crates/ty_ide/src/django_template.rs create mode 100644 crates/ty_ide/src/django_template/builtins.rs create mode 100644 crates/ty_ide/src/django_template/completion.rs create mode 100644 crates/ty_ide/src/django_template/goto.rs create mode 100644 crates/ty_ide/src/django_template/index.rs create mode 100644 crates/ty_ide/src/django_template/lexer.rs create mode 100644 crates/ty_ide/src/django_template/project.rs create mode 100644 crates/ty_ide/src/django_template/resolve.rs create mode 100644 crates/ty_ide/src/django_template/semantic_tokens.rs create mode 100644 crates/ty_server/tests/e2e/django_templates.rs create mode 100644 docs/basedpython/frameworks/django-templates.md diff --git a/crates/ty_ide/src/django_template.rs b/crates/ty_ide/src/django_template.rs new file mode 100644 index 0000000000..8f20052d06 --- /dev/null +++ b/crates/ty_ide/src/django_template.rs @@ -0,0 +1,232 @@ +//! ide support for django template files +//! +//! django templates are not python, so none of the machinery the rest of this +//! crate is built on — the parser, the semantic index, type inference — applies +//! to them. what *is* shared is the project: a template's variables come from +//! the view that renders it, its `{% url %}` names come from the project's url +//! configuration, and its custom tags and filters come from the project's +//! `templatetags` modules. so this module owns a small template front end of its +//! own ([`lexer`], [`index`]) and spends the rest of its effort joining it to the +//! python side ([`project`]). + +mod builtins; +mod completion; +mod goto; +mod index; +mod lexer; +mod project; +mod resolve; +mod semantic_tokens; + +pub use completion::{TemplateCompletion, TemplateEdit}; + +use ruff_db::files::File; +use ruff_db::source::source_text; +use ruff_db::system::SystemPath; +use ruff_text_size::{TextRange, TextSize}; +use ty_project::Db; + +use crate::semantic_tokens::SemanticTokens; +use crate::{NavigationTargets, RangedValue}; + +use index::TemplateIndex; + +/// the file extensions a django template is conventionally written with +/// +/// django itself puts no constraint on the extension — the template loader takes +/// whatever path it is given — so this list only decides what the server will +/// *offer* template support for when the editor hasn't already told it the file +/// is a template. +const TEMPLATE_EXTENSIONS: &[&str] = &["html", "htm", "txt", "xml", "django", "dj", "jinja"]; + +/// the directory name django's app-directories loader looks in +const TEMPLATE_DIRECTORY: &str = "templates"; + +/// whether `path` looks like a django template +/// +/// an editor that knows the file's language (vs code's `django-html`, vim's +/// `htmldjango`) tells the server directly and this is never consulted. it is +/// the fallback for the much more common case of a `.html` file that the editor +/// reports as plain html, and it deliberately requires the file to be *inside* a +/// `templates` directory so that ordinary html in a project is left alone. +pub fn is_django_template_path(path: &SystemPath) -> bool { + let has_template_extension = path + .extension() + .is_some_and(|extension| TEMPLATE_EXTENSIONS.contains(&extension)); + + has_template_extension + && path + .ancestors() + .any(|ancestor| ancestor.file_name() == Some(TEMPLATE_DIRECTORY)) +} + +/// the index of `file`, parsed as a django template +/// +/// the query is tracked so that the several ide features that need it — the +/// completions, the semantic tokens, goto — parse each template once per edit +/// rather than once per request. +#[salsa::tracked(returns(ref), heap_size=ruff_memory_usage::heap_size)] +fn template_index(db: &dyn Db, file: File) -> TemplateIndex { + TemplateIndex::from_source(source_text(db, file).as_str()) +} + +/// the semantic tokens of `file`, read as a django template +/// +/// `range` restricts the result to the tokens it touches, for the ranged +/// semantic-tokens request. +pub fn django_template_semantic_tokens( + db: &dyn Db, + file: File, + range: Option, +) -> SemanticTokens { + let source = source_text(db, file); + SemanticTokens::new(semantic_tokens::semantic_tokens( + template_index(db, file), + source.as_str(), + range, + )) +} + +/// where the name at `offset` of `file` is defined, read as a django template +pub fn django_template_goto_definition( + db: &dyn Db, + file: File, + offset: TextSize, +) -> Option> { + let source = source_text(db, file); + goto::goto_definition(db, file, template_index(db, file), source.as_str(), offset) +} + +/// the completions for `offset` in `file`, read as a django template +pub fn django_template_completions( + db: &dyn Db, + file: File, + offset: TextSize, +) -> Vec { + let source = source_text(db, file); + completion::completions(db, file, template_index(db, file), source.as_str(), offset) +} + +#[cfg(test)] +pub(crate) mod tests { + use ruff_db::files::{File, system_path_to_file}; + use ruff_db::system::{DbWithWritableSystem, SystemPath, SystemPathBuf}; + use ruff_python_ast::PythonVersion; + use ruff_python_trivia::textwrap::dedent; + use ruff_text_size::TextSize; + use ty_project::{ProjectMetadata, TestDb}; + + use super::{ + django_template_completions, django_template_goto_definition, is_django_template_path, + }; + + /// a project whose files are written out, with the cursor marked by + /// `` in exactly one of them + pub(crate) struct TemplateTest { + pub(crate) db: TestDb, + pub(crate) file: File, + pub(crate) offset: TextSize, + } + + impl TemplateTest { + /// build a project from `(path, contents)` pairs + pub(crate) fn new(sources: &[(&str, &str)]) -> Self { + const MARKER: &str = ""; + + let mut db = TestDb::new(ProjectMetadata::new("test", SystemPathBuf::from("/"))); + db.init_program_with_python_version(PythonVersion::latest_ty()) + .unwrap(); + + let mut cursor = None; + + for (path, contents) in sources { + let contents = dedent(contents).into_owned(); + + let (contents, offset) = match contents.find(MARKER) { + Some(index) => { + let mut without = contents[..index].to_string(); + without.push_str(&contents[index + MARKER.len()..]); + (without, Some(TextSize::try_from(index).unwrap())) + } + None => (contents, None), + }; + + db.write_file(path, &contents).unwrap(); + let file = system_path_to_file(&db, path).unwrap(); + + if let Some(offset) = offset { + assert!(cursor.is_none(), "more than one `` marker"); + cursor = Some((file, offset)); + } + } + + let (file, offset) = cursor.expect("a source to contain ``"); + Self { db, file, offset } + } + + /// the labels of the completions at the cursor, in the order offered + pub(crate) fn completions(&self) -> Vec { + django_template_completions(&self.db, self.file, self.offset) + .into_iter() + .map(|completion| completion.label) + .collect() + } + + /// the completions at the cursor, rendered as `label — detail` + pub(crate) fn detailed(&self) -> Vec { + django_template_completions(&self.db, self.file, self.offset) + .into_iter() + .map(|completion| match completion.detail { + Some(detail) => format!("{} — {detail}", completion.label), + None => completion.label, + }) + .collect() + } + + /// where goto-definition at the cursor lands, as `path:text` + pub(crate) fn definitions(&self) -> Vec { + let Some(targets) = django_template_goto_definition(&self.db, self.file, self.offset) + else { + return Vec::new(); + }; + + targets + .into_iter() + .map(|target| { + let source = ruff_db::source::source_text(&self.db, target.file()); + format!( + "{}:{}", + // the memory file system reports the host's separator + target.file().path(&self.db).to_string().replace('\\', "/"), + &source[target.focus_range()] + ) + }) + .collect() + } + } + + #[test] + fn a_template_is_an_html_file_under_a_templates_directory() { + assert!(is_django_template_path(SystemPath::new( + "/app/templates/blog/post.html" + ))); + assert!(is_django_template_path(SystemPath::new( + "/templates/base.txt" + ))); + } + + #[test] + fn ordinary_html_outside_a_templates_directory_is_not_a_template() { + assert!(!is_django_template_path(SystemPath::new( + "/app/static/index.html" + ))); + assert!(!is_django_template_path(SystemPath::new("/README.html"))); + } + + #[test] + fn a_python_file_is_never_a_template() { + assert!(!is_django_template_path(SystemPath::new( + "/app/templates/views.py" + ))); + } +} diff --git a/crates/ty_ide/src/django_template/builtins.rs b/crates/ty_ide/src/django_template/builtins.rs new file mode 100644 index 0000000000..0bee7ac9f8 --- /dev/null +++ b/crates/ty_ide/src/django_template/builtins.rs @@ -0,0 +1,769 @@ +//! django's builtin template tags and filters +//! +//! these are the ones `django.template.defaulttags`, `django.template.defaultfilters` +//! and the libraries shipped in `django.templatetags` register. a project's own +//! tags and filters are discovered from its source instead — see [`super::project`]. +//! +//! the tables carry the block structure (which tag closes which, and which tags +//! may appear in between) because the completions and the index both need it: a +//! `{% for %}` without the knowledge that `{% empty %}` belongs inside it would +//! either close the block early or never offer `{% empty %}` at all. + +/// a builtin template tag +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Tag { + pub(crate) name: &'static str, + /// the tag closing the block this one opens, for a block tag + pub(crate) closed_by: Option<&'static str>, + /// the tags that may appear between this tag and the one that closes it + pub(crate) branches: &'static [&'static str], + /// the `{% load %}` library providing this tag, or `None` when it is always + /// available + pub(crate) library: Option<&'static str>, + pub(crate) documentation: &'static str, +} + +/// a builtin template filter +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Filter { + pub(crate) name: &'static str, + /// the `{% load %}` library providing this filter, or `None` when it is + /// always available + pub(crate) library: Option<&'static str>, + pub(crate) documentation: &'static str, +} + +/// look a builtin tag up by name +pub(crate) fn tag(name: &str) -> Option<&'static Tag> { + TAGS.iter().find(|tag| tag.name == name) +} + +/// look a builtin filter up by name +pub(crate) fn filter(name: &str) -> Option<&'static Filter> { + FILTERS.iter().find(|filter| filter.name == name) +} + +/// the tag closing the block `name` opens, whether `name` is builtin or one of +/// the project's own block tags +pub(crate) fn end_tag_for(name: &str) -> Option<&'static str> { + tag(name).and_then(|tag| tag.closed_by) +} + +/// the libraries a `{% load %}` can name +pub(crate) const LIBRARIES: &[&str] = &["cache", "i18n", "l10n", "static", "tz"]; + +pub(crate) const TAGS: &[Tag] = &[ + Tag { + name: "autoescape", + closed_by: Some("endautoescape"), + branches: &[], + library: None, + documentation: "controls html auto-escaping for the enclosed block. takes `on` or `off`.", + }, + Tag { + name: "block", + closed_by: Some("endblock"), + branches: &[], + library: None, + documentation: "defines a named block a child template can override.", + }, + Tag { + name: "comment", + closed_by: Some("endcomment"), + branches: &[], + library: None, + documentation: "ignores everything between the tags.", + }, + Tag { + name: "csrf_token", + closed_by: None, + branches: &[], + library: None, + documentation: "renders the hidden csrf token input. required in every `POST` form.", + }, + Tag { + name: "cycle", + closed_by: None, + branches: &[], + library: None, + documentation: "emits the next of its arguments each time it is reached.", + }, + Tag { + name: "debug", + closed_by: None, + branches: &[], + library: None, + documentation: "outputs the whole current context, for debugging.", + }, + Tag { + name: "extends", + closed_by: None, + branches: &[], + library: None, + documentation: "declares this template a child of another. must be the first tag in the file.", + }, + Tag { + name: "filter", + closed_by: Some("endfilter"), + branches: &[], + library: None, + documentation: "runs the enclosed block's output through the given filters.", + }, + Tag { + name: "firstof", + closed_by: None, + branches: &[], + library: None, + documentation: "outputs the first of its arguments that is truthy.", + }, + Tag { + name: "for", + closed_by: Some("endfor"), + branches: &["empty"], + library: None, + documentation: "loops over each item of a sequence. `{% empty %}` supplies the body for an empty one.", + }, + Tag { + name: "if", + closed_by: Some("endif"), + branches: &["elif", "else"], + library: None, + documentation: "renders its body when the condition is truthy.", + }, + Tag { + name: "ifchanged", + closed_by: Some("endifchanged"), + branches: &["else"], + library: None, + documentation: "renders its body only when the value has changed since the last loop iteration.", + }, + Tag { + name: "include", + closed_by: None, + branches: &[], + library: None, + documentation: "renders another template here, with the current context or a `with` one.", + }, + Tag { + name: "load", + closed_by: None, + branches: &[], + library: None, + documentation: "loads a template tag library, making its tags and filters available.", + }, + Tag { + name: "lorem", + closed_by: None, + branches: &[], + library: None, + documentation: "emits placeholder lorem ipsum text.", + }, + Tag { + name: "now", + closed_by: None, + branches: &[], + library: None, + documentation: "formats the current date and time with the given format string.", + }, + Tag { + name: "partial", + closed_by: None, + branches: &[], + library: None, + documentation: "renders a fragment defined by a `{% partialdef %}`.", + }, + Tag { + name: "partialdef", + closed_by: Some("endpartialdef"), + branches: &[], + library: None, + documentation: "defines a named, reusable fragment of this template. `inline` also renders it in place.", + }, + Tag { + name: "querystring", + closed_by: None, + branches: &[], + library: None, + documentation: "renders a url-encoded query string from the request's, with the given changes applied.", + }, + Tag { + name: "regroup", + closed_by: None, + branches: &[], + library: None, + documentation: "regroups a list of objects by a common attribute.", + }, + Tag { + name: "resetcycle", + closed_by: None, + branches: &[], + library: None, + documentation: "restarts a `{% cycle %}` from its first argument.", + }, + Tag { + name: "spaceless", + closed_by: Some("endspaceless"), + branches: &[], + library: None, + documentation: "strips the whitespace between html tags in its body.", + }, + Tag { + name: "templatetag", + closed_by: None, + branches: &[], + library: None, + documentation: "outputs one of the template language's own delimiters, such as `openblock`.", + }, + Tag { + name: "url", + closed_by: None, + branches: &[], + library: None, + documentation: "reverses a named url pattern into its path.", + }, + Tag { + name: "verbatim", + closed_by: Some("endverbatim"), + branches: &[], + library: None, + documentation: "outputs its body without rendering any template syntax in it.", + }, + Tag { + name: "widthratio", + closed_by: None, + branches: &[], + library: None, + documentation: "scales a value against a maximum, for bar-chart widths.", + }, + Tag { + name: "with", + closed_by: Some("endwith"), + branches: &[], + library: None, + documentation: "binds names to values for the enclosed block.", + }, + // `cache` + Tag { + name: "cache", + closed_by: Some("endcache"), + branches: &[], + library: Some("cache"), + documentation: "caches the rendered body for the given number of seconds, keyed by the given name.", + }, + // `i18n` + Tag { + name: "blocktranslate", + closed_by: Some("endblocktranslate"), + branches: &["plural"], + library: Some("i18n"), + documentation: "marks a block of text for translation, with placeholders for variables.", + }, + Tag { + name: "blocktrans", + closed_by: Some("endblocktrans"), + branches: &["plural"], + library: Some("i18n"), + documentation: "the older spelling of `{% blocktranslate %}`.", + }, + Tag { + name: "get_available_languages", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "binds the list of configured `(code, name)` language pairs to a variable.", + }, + Tag { + name: "get_current_language", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "binds the active language's code to a variable.", + }, + Tag { + name: "get_current_language_bidi", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "binds whether the active language is right-to-left to a variable.", + }, + Tag { + name: "get_language_info", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "binds a language's name, code and direction to a variable.", + }, + Tag { + name: "get_language_info_list", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "binds the language info of each of the given codes to a variable.", + }, + Tag { + name: "language", + closed_by: Some("endlanguage"), + branches: &[], + library: Some("i18n"), + documentation: "renders its body with the given language active.", + }, + Tag { + name: "translate", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "translates a string literal or variable.", + }, + Tag { + name: "trans", + closed_by: None, + branches: &[], + library: Some("i18n"), + documentation: "the older spelling of `{% translate %}`.", + }, + // `l10n` + Tag { + name: "localize", + closed_by: Some("endlocalize"), + branches: &[], + library: Some("l10n"), + documentation: "turns locale-aware number formatting on or off for its body.", + }, + // `static` + Tag { + name: "get_media_prefix", + closed_by: None, + branches: &[], + library: Some("static"), + documentation: "binds `MEDIA_URL` to a variable.", + }, + Tag { + name: "get_static_prefix", + closed_by: None, + branches: &[], + library: Some("static"), + documentation: "binds `STATIC_URL` to a variable.", + }, + Tag { + name: "static", + closed_by: None, + branches: &[], + library: Some("static"), + documentation: "builds the url of a static file.", + }, + // `tz` + Tag { + name: "get_current_timezone", + closed_by: None, + branches: &[], + library: Some("tz"), + documentation: "binds the active time zone's name to a variable.", + }, + Tag { + name: "localtime", + closed_by: Some("endlocaltime"), + branches: &[], + library: Some("tz"), + documentation: "turns conversion of datetimes to local time on or off for its body.", + }, + Tag { + name: "timezone", + closed_by: Some("endtimezone"), + branches: &[], + library: Some("tz"), + documentation: "renders its body with the given time zone active.", + }, +]; + +pub(crate) const FILTERS: &[Filter] = &[ + Filter { + name: "add", + library: None, + documentation: "adds the argument to the value.", + }, + Filter { + name: "addslashes", + library: None, + documentation: "backslash-escapes quotes.", + }, + Filter { + name: "capfirst", + library: None, + documentation: "upper-cases the first character.", + }, + Filter { + name: "center", + library: None, + documentation: "centres the value in a field of the given width.", + }, + Filter { + name: "cut", + library: None, + documentation: "removes every occurrence of the argument.", + }, + Filter { + name: "date", + library: None, + documentation: "formats a date with the given format string.", + }, + Filter { + name: "default", + library: None, + documentation: "uses the argument when the value is falsy.", + }, + Filter { + name: "default_if_none", + library: None, + documentation: "uses the argument only when the value is `None`.", + }, + Filter { + name: "dictsort", + library: None, + documentation: "sorts a list of mappings by the given key.", + }, + Filter { + name: "dictsortreversed", + library: None, + documentation: "sorts a list of mappings by the given key, descending.", + }, + Filter { + name: "divisibleby", + library: None, + documentation: "whether the value divides by the argument.", + }, + Filter { + name: "escape", + library: None, + documentation: "html-escapes the value.", + }, + Filter { + name: "escapejs", + library: None, + documentation: "escapes the value for use in a javascript string.", + }, + Filter { + name: "escapeseq", + library: None, + documentation: "html-escapes each element of a sequence.", + }, + Filter { + name: "filesizeformat", + library: None, + documentation: "formats a byte count as `13 KB`.", + }, + Filter { + name: "first", + library: None, + documentation: "the first element.", + }, + Filter { + name: "floatformat", + library: None, + documentation: "rounds a float to the given number of decimal places.", + }, + Filter { + name: "force_escape", + library: None, + documentation: "html-escapes the value immediately rather than lazily.", + }, + Filter { + name: "get_digit", + library: None, + documentation: "the nth digit of an integer, counted from the right.", + }, + Filter { + name: "iriencode", + library: None, + documentation: "converts an iri to a url-safe string.", + }, + Filter { + name: "join", + library: None, + documentation: "joins a sequence with the argument, like python's `str.join`.", + }, + Filter { + name: "json_script", + library: None, + documentation: "renders the value as json inside a `