From 09979034bf48f055ec387b661f04c09cdeac8289 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 10:48:26 -0400 Subject: [PATCH 01/25] perf: rewrite to lexer architecture --- src/lib/lexer.ts | 439 ++++++++ src/lib/lexer_bash.ts | 706 +++++++++++++ src/lib/lexer_css.ts | 340 ++++++ src/lib/lexer_json.ts | 156 +++ src/lib/lexer_ts.ts | 988 ++++++++++++++++++ src/lib/syntax_styler.ts | 56 +- src/lib/syntax_styler_global.ts | 12 + .../fixtures/generated/bash/bash_complex.html | 16 +- .../fixtures/generated/bash/bash_complex.txt | 50 +- .../fixtures/generated/css/css_complex.html | 4 +- .../fixtures/generated/css/css_complex.txt | 10 +- .../fixtures/generated/json/json_complex.html | 8 +- .../fixtures/generated/json/json_complex.txt | 22 +- .../fixtures/generated/ts/ts_complex.html | 92 +- src/test/fixtures/generated/ts/ts_complex.txt | 175 ++-- src/test/fixtures/helpers.ts | 5 + src/test/lexer.test.ts | 178 ++++ src/test/lexer_bash.test.ts | 215 ++++ src/test/lexer_css.test.ts | 146 +++ src/test/lexer_json.test.ts | 92 ++ src/test/lexer_ts.test.ts | 273 +++++ 21 files changed, 3750 insertions(+), 233 deletions(-) create mode 100644 src/lib/lexer.ts create mode 100644 src/lib/lexer_bash.ts create mode 100644 src/lib/lexer_css.ts create mode 100644 src/lib/lexer_json.ts create mode 100644 src/lib/lexer_ts.ts create mode 100644 src/test/lexer.test.ts create mode 100644 src/test/lexer_bash.test.ts create mode 100644 src/test/lexer_css.test.ts create mode 100644 src/test/lexer_json.test.ts create mode 100644 src/test/lexer_ts.test.ts diff --git a/src/lib/lexer.ts b/src/lib/lexer.ts new file mode 100644 index 00000000..4c3e3036 --- /dev/null +++ b/src/lib/lexer.ts @@ -0,0 +1,439 @@ +/** + * Flat-event lexer substrate — the engine beneath the hand-written per-language + * lexers (`lexer_json.ts`, `lexer_ts.ts`, …), replacing the regex-grammar engine + * in `tokenize_syntax.ts`. + * + * Tokens are emitted as variable-length records into one `Int32Array`: + * + * - leaf: `[type_id, start, end]` where `type_id > 0` + * - open: `[-type_id, start]` — opens a container token + * - close: `[0, end]` — closes the innermost open container + * + * Untyped text is implicit — it's the gap between events, recovered from + * offsets. Adjacent same-type leaves are coalesced at emit time so runs like + * `});` become one span. + * + * @module + */ + +/** + * Interned metadata for a token type, shared by all lexers and stylers. + */ +export interface SyntaxTypeInfo { + id: number; + name: string; + aliases: Array; + /** + * Space-separated CSS classes, e.g. `'token_null token_keyword'`. + */ + classes: string; + /** + * Precomputed HTML open tag, e.g. `''`. + */ + open_tag: string; +} + +const type_infos: Array = [ + // id 0 is reserved — it's the close-event tag + {id: 0, name: '', aliases: [], classes: '', open_tag: ''}, +]; +const type_ids: Map = new Map(); + +/** + * Interns a token type by name (+ optional aliases) into the global id space + * and returns its id. Repeated calls with the same name and aliases return the + * same id. The CSS class list and HTML open tag are precomputed here so + * emitters never build class strings at runtime. + */ +export const token_type = (name: string, alias?: string | Array): number => { + const aliases = alias === undefined ? [] : Array.isArray(alias) ? alias : [alias]; + const key = name + '\x1F' + aliases.join('\x1F'); + const existing = type_ids.get(key); + if (existing !== undefined) return existing; + let classes = 'token_' + name; + for (const a of aliases) classes += ' token_' + a; + const id = type_infos.length; + type_infos.push({id, name, aliases, classes, open_tag: ''}); + type_ids.set(key, id); + return id; +}; + +/** + * Looks up the interned info for a token type id. + */ +export const token_type_info = (id: number): SyntaxTypeInfo => type_infos[id]!; + +/** + * A lexer-based language registration — the replacement for regex grammars. + */ +export interface SyntaxLang { + /** + * Primary language id, e.g. `'ts'`. + */ + id: string; + /** + * Alternate ids resolving to this language, e.g. `['typescript']`. + */ + aliases?: Array; + /** + * Lexes the `lexer`'s current `[pos, end)` window, emitting token events. + * Must never throw and must always terminate with `lexer.pos === lexer.end`. + */ + lex: (lexer: Lexer) => void; +} + +/** + * The result of lexing: the source text plus its flat token event stream. + */ +export interface LexedSyntax { + text: string; + events: Int32Array; + events_len: number; +} + +/** + * Shared lexing context passed to language lex functions. Holds the text + * window, the event buffer, and the language registry for embedding. + */ +export class Lexer { + text = ''; + pos = 0; + end = 0; + /** + * Language registry for `embed` — set by `lex_syntax`. + */ + langs: Map | null = null; + + events: Int32Array; + events_len = 0; + + // index of the previous event iff it was a leaf, for adjacency coalescing + #last_leaf = -1; + + constructor(capacity = 256) { + this.events = new Int32Array(capacity < 256 ? 256 : capacity); + } + + #ensure(extra: number): void { + if (this.events_len + extra > this.events.length) { + const next = new Int32Array(this.events.length * 2); + next.set(this.events); + this.events = next; + } + } + + /** + * Emits a leaf token. Empty spans are dropped; a leaf adjacent to a + * preceding leaf of the same type extends it instead (span coalescing). + * + * @mutates `this` + */ + leaf(type_id: number, start: number, end: number): void { + if (start >= end) return; + const i = this.#last_leaf; + const {events} = this; + if (i !== -1 && events[i] === type_id && events[i + 2] === start) { + events[i + 2] = end; + return; + } + this.#ensure(3); + const at = this.events_len; + this.events[at] = type_id; + this.events[at + 1] = start; + this.events[at + 2] = end; + this.events_len = at + 3; + this.#last_leaf = at; + } + + /** + * Opens a container token at `start`. Must be balanced by a later `close`. + * + * @mutates `this` + */ + open(type_id: number, start: number): void { + this.#ensure(2); + const at = this.events_len; + this.events[at] = -type_id; + this.events[at + 1] = start; + this.events_len = at + 2; + this.#last_leaf = -1; + } + + /** + * Closes the innermost open container at `end`. + * + * @mutates `this` + */ + close(end: number): void { + this.#ensure(2); + const at = this.events_len; + this.events[at] = 0; + this.events[at + 1] = end; + this.events_len = at + 2; + this.#last_leaf = -1; + } + + /** + * Lexes `[start, end)` with the language registered as `lang_id`, + * restoring this lexer's window afterward. Returns `false` (leaving the + * region as plain text) when the language isn't registered. + * + * @mutates `this` + */ + embed(lang_id: string, start: number, end: number): boolean { + if (start >= end) return false; + const lang = this.langs?.get(lang_id); + if (!lang) return false; + const prev_pos = this.pos; + const prev_end = this.end; + this.pos = start; + this.end = end; + lang.lex(this); + this.pos = prev_pos; + this.end = prev_end; + this.#last_leaf = -1; + return true; + } +} + +/** + * Lexes `text` with `lang`, returning the flat token event stream. + * + * @param text - the source text + * @param lang - the language to lex with + * @param langs - registry used to resolve embedded languages by id + */ +export const lex_syntax = ( + text: string, + lang: SyntaxLang, + langs?: Map, +): LexedSyntax => { + // capacity heuristic: dense token streams run ~1 int per source char + const lexer = new Lexer(text.length); + lexer.text = text; + lexer.pos = 0; + lexer.end = text.length; + lexer.langs = langs ?? null; + lang.lex(lexer); + return {text, events: lexer.events, events_len: lexer.events_len}; +}; + +/** + * Escapes `text[from..to)` for HTML text content in a single pass. + * Only `&`, `<`, and non-breaking spaces need handling (nbsp normalizes to a + * regular space, matching the old engine's policy). + */ +const escape_html_slice = (text: string, from: number, to: number): string => { + let out = ''; + let seg = from; + for (let i = from; i < to; i++) { + const c = text.charCodeAt(i); + if (c === 38) { + out += text.slice(seg, i) + '&'; + seg = i + 1; + } else if (c === 60) { + out += text.slice(seg, i) + '<'; + seg = i + 1; + } else if (c === 160) { + out += text.slice(seg, i) + ' '; + seg = i + 1; + } + } + return seg === from ? text.slice(from, to) : out + text.slice(seg, to); +}; + +/** + * Renders a lexed token event stream to HTML in one forward pass. + * Gap text is copy-escaped; token spans use the precomputed open tags. + */ +export const render_syntax_html = (lexed: LexedSyntax): string => { + const {text, events, events_len} = lexed; + let out = ''; + let pos = 0; + let i = 0; + while (i < events_len) { + const tag = events[i]!; + if (tag > 0) { + const start = events[i + 1]!; + const end = events[i + 2]!; + if (start > pos) out += escape_html_slice(text, pos, start); + out += type_infos[tag]!.open_tag + escape_html_slice(text, start, end) + ''; + pos = end; + i += 3; + } else if (tag < 0) { + const start = events[i + 1]!; + if (start > pos) out += escape_html_slice(text, pos, start); + out += type_infos[-tag]!.open_tag; + pos = start; + i += 2; + } else { + const end = events[i + 1]!; + if (end > pos) out += escape_html_slice(text, pos, end); + out += ''; + pos = end; + i += 2; + } + } + if (pos < text.length) out += escape_html_slice(text, pos, text.length); + return out; +}; + +/** + * A flattened token span, in document order with containers before their + * children. Used by fixtures, tests, and range building. + */ +export interface SyntaxEventToken { + type: string; + start: number; + end: number; +} + +/** + * Flattens a lexed event stream to `SyntaxEventToken`s in document order + * (containers precede their children). + */ +export const syntax_events_to_tokens = (lexed: LexedSyntax): Array => { + const {events, events_len} = lexed; + const tokens: Array = []; + const stack: Array = []; + let i = 0; + while (i < events_len) { + const tag = events[i]!; + if (tag > 0) { + tokens.push({type: type_infos[tag]!.name, start: events[i + 1]!, end: events[i + 2]!}); + i += 3; + } else if (tag < 0) { + const t: SyntaxEventToken = {type: type_infos[-tag]!.name, start: events[i + 1]!, end: -1}; + tokens.push(t); + stack.push(t); + i += 2; + } else { + const t = stack.pop(); + if (t) t.end = events[i + 1]!; + i += 2; + } + } + return tokens; +}; + +/** + * Validates a lexed event stream's structural invariants, returning a list of + * human-readable issues (empty when valid): records well-formed, offsets + * monotonic and in-bounds, containers balanced. + */ +export const validate_syntax_events = (lexed: LexedSyntax): Array => { + const {text, events, events_len} = lexed; + const issues: Array = []; + const stack: Array = []; + let cursor = 0; + let i = 0; + while (i < events_len) { + const tag = events[i]!; + if (tag > 0) { + if (i + 3 > events_len) { + issues.push(`truncated leaf record at ${i}`); + break; + } + const start = events[i + 1]!; + const end = events[i + 2]!; + if (tag >= type_infos.length) issues.push(`unknown type id ${tag} at ${i}`); + if (start < cursor) issues.push(`leaf start ${start} overlaps cursor ${cursor} at ${i}`); + if (end <= start) issues.push(`empty or inverted leaf [${start}, ${end}) at ${i}`); + if (end > text.length) issues.push(`leaf end ${end} out of bounds at ${i}`); + cursor = end; + i += 3; + } else if (tag < 0) { + if (i + 2 > events_len) { + issues.push(`truncated open record at ${i}`); + break; + } + const start = events[i + 1]!; + if (-tag >= type_infos.length) issues.push(`unknown type id ${-tag} at ${i}`); + if (start < cursor) issues.push(`open start ${start} overlaps cursor ${cursor} at ${i}`); + if (start > text.length) issues.push(`open start ${start} out of bounds at ${i}`); + stack.push(start); + cursor = start; + i += 2; + } else { + if (i + 2 > events_len) { + issues.push(`truncated close record at ${i}`); + break; + } + const end = events[i + 1]!; + const open_start = stack.pop(); + if (open_start === undefined) { + issues.push(`close without open at ${i}`); + } else if (end < open_start) { + issues.push(`container inverted: open ${open_start}, close ${end} at ${i}`); + } + if (end < cursor) issues.push(`close end ${end} before cursor ${cursor} at ${i}`); + if (end > text.length) issues.push(`close end ${end} out of bounds at ${i}`); + cursor = end; + i += 2; + } + } + if (stack.length > 0) issues.push(`${stack.length} unclosed container(s)`); + return issues; +}; + +// Shared ASCII char-class flags. Code units >= 0xa0 are identifier chars by +// policy (matches the inherited U+00A0-U+FFFF identifier ranges; surrogate +// halves are >= 0xa0 so astral chars behave consistently). +export const CF_SPACE = 1; +export const CF_DIGIT = 2; +export const CF_IDENT_START = 4; +export const CF_IDENT = 8; + +export const CHAR_FLAGS: Uint8Array = new Uint8Array(128); +for (let c = 48; c <= 57; c++) CHAR_FLAGS[c] = CF_DIGIT | CF_IDENT; // 0-9 +for (let c = 65; c <= 90; c++) CHAR_FLAGS[c] = CF_IDENT_START | CF_IDENT; // A-Z +for (let c = 97; c <= 122; c++) CHAR_FLAGS[c] = CF_IDENT_START | CF_IDENT; // a-z +CHAR_FLAGS[36] = CF_IDENT_START | CF_IDENT; // $ +CHAR_FLAGS[95] = CF_IDENT_START | CF_IDENT; // _ +CHAR_FLAGS[9] = CF_SPACE; // \t +CHAR_FLAGS[10] = CF_SPACE; // \n +CHAR_FLAGS[11] = CF_SPACE; // \v +CHAR_FLAGS[12] = CF_SPACE; // \f +CHAR_FLAGS[13] = CF_SPACE; // \r +CHAR_FLAGS[32] = CF_SPACE; // space + +export const is_space = (c: number): boolean => c < 128 && (CHAR_FLAGS[c]! & CF_SPACE) !== 0; + +export const is_digit = (c: number): boolean => c >= 48 && c <= 57; + +export const is_ident_start = (c: number): boolean => + c < 128 ? (CHAR_FLAGS[c]! & CF_IDENT_START) !== 0 : c >= 0xa0; + +export const is_ident = (c: number): boolean => + c < 128 ? (CHAR_FLAGS[c]! & CF_IDENT) !== 0 : c >= 0xa0; + +/** + * Scans an identifier starting at `from` (assumed to be an identifier start), + * returning the exclusive end index. + */ +export const scan_ident = (text: string, from: number, end: number): number => { + let i = from + 1; + while (i < end && is_ident(text.charCodeAt(i))) i++; + return i; +}; + +/** + * Skips whitespace (including newlines) from `from`, returning the next + * non-space index. + */ +export const skip_space = (text: string, from: number, end: number): number => { + let i = from; + while (i < end && is_space(text.charCodeAt(i))) i++; + return i; +}; + +/** + * Returns the index of the next `\n` at or after `i` (exclusive end of the + * line's content, excluding a preceding `\r`), or `end` when there is none. + * Uses native `indexOf` — the fast path for line-oriented scans. + */ +export const scan_to_line_end = (text: string, i: number, end: number): number => { + const nl = text.indexOf('\n', i); + if (nl === -1 || nl >= end) return end; + return nl > i && text.charCodeAt(nl - 1) === 13 ? nl - 1 : nl; +}; diff --git a/src/lib/lexer_bash.ts b/src/lib/lexer_bash.ts new file mode 100644 index 00000000..178fcde3 --- /dev/null +++ b/src/lib/lexer_bash.ts @@ -0,0 +1,706 @@ +import { + is_digit, + is_space, + scan_to_line_end, + skip_space, + token_type, + type Lexer, + type SyntaxLang, +} from './lexer.ts'; + +/** + * Hand-written Bash/shell lexer. + * + * Token vocabulary matches the retired regex grammar: `shebang`, `comment`, + * `string` (double-quoted strings are containers whose `$`-expansions nest as + * `variable`/`command_substitution`), `keyword`, `builtin`, `boolean`, + * `number`, `variable`, `command_substitution` (a container), `function`, + * `file_descriptor`, `operator`, `punctuation`, plus heredocs (`heredoc` + * container + `heredoc_delimiter`). + * + * Fidelity fixes over the regex model: + * - arithmetic expansion `$((…))` is recognized as distinct from command + * substitution `$(…)`; its `$((`/`))` are punctuation and the interior lexes + * as ordinary bash (numbers, `$vars`, and general operators fall out + * naturally — no dedicated arithmetic token types). + * - heredocs match any delimiter, honor `<<-`, and support quoted (no + * expansion) vs unquoted (expanded) bodies; multiple heredocs redirected on + * one line are queued and their bodies consumed in order. + * + * Word classification (keyword/builtin/boolean) is a context-free `Map` + * lookup, matching the old grammar's type-major behavior. + * + * Resilience: unterminated single-line constructs stop at the line boundary; + * unterminated strings, command substitutions, and heredocs extend to the + * window end. + */ + +const T_SHEBANG = token_type('shebang', 'comment'); +const T_COMMENT = token_type('comment'); +const T_STRING = token_type('string'); +const T_KEYWORD = token_type('keyword'); +const T_BUILTIN = token_type('builtin'); +const T_BOOLEAN = token_type('boolean'); +const T_NUMBER = token_type('number'); +const T_VARIABLE = token_type('variable'); +const T_COMMAND_SUBSTITUTION = token_type('command_substitution'); +const T_FUNCTION = token_type('function'); +const T_FILE_DESCRIPTOR = token_type('file_descriptor', 'important'); +const T_OPERATOR = token_type('operator'); +const T_PUNCTUATION = token_type('punctuation'); +const T_HEREDOC = token_type('heredoc', 'string'); +const T_HEREDOC_DELIMITER = token_type('heredoc_delimiter', 'punctuation'); + +const K_KEYWORD = 1; +const K_BUILTIN = 2; +const K_BOOLEAN = 3; + +const WORDS: Map = new Map(); +const add_words = (kind: number, words: string): void => { + for (const word of words.split(' ')) WORDS.set(word, kind); +}; +add_words( + K_KEYWORD, + 'if then else elif fi for while until do done case esac in select function return local ' + + 'export declare typeset readonly unset set shift trap break continue coproc time', +); +add_words( + K_BUILTIN, + 'echo printf cd pwd read test source eval exec exit getopts hash type ulimit umask wait kill ' + + 'jobs bg fg disown alias unalias command shopt', +); +add_words(K_BOOLEAN, 'true false'); + +// `$`-followed single-char special variables (`$@ $# $? $$ $! $* $-`); digits +// are handled by the `$word` scan +const SPECIAL_VAR: Set = new Set([33, 64, 35, 36, 42, 63, 45]); + +const is_word_char = (c: number): boolean => + (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || (c >= 48 && c <= 57) || c === 95; + +const is_alnum = (c: number): boolean => + (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || (c >= 48 && c <= 57); + +const is_hex = (c: number): boolean => + (c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70); + +const skip_blank = (text: string, from: number, end: number): number => { + let i = from; + while (i < end && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9)) i++; + return i; +}; + +// index of the next `\n` at or after `i` (or `end`) — for stepping between +// heredoc lines. Distinct from `scan_to_line_end`, which stops *before* a +// trailing `\r` and is used for single-line token boundaries. +const line_end_of = (text: string, i: number, end: number): number => { + const nl = text.indexOf('\n', i); + return nl === -1 || nl >= end ? end : nl; +}; + +/** + * Scans a bash numeric literal from the digit at `i`: hex (`0x…`), base-N + * (`\d+#[alnum]+`), or plain decimal (`\d+`, covering octal). Returns the + * exclusive end. + */ +const scan_bash_number = (text: string, i: number, end: number): number => { + if (text.charCodeAt(i) === 48 && (text.charCodeAt(i + 1) | 0x20) === 120) { + let j = i + 2; + while (j < end && is_hex(text.charCodeAt(j))) j++; + return j; + } + let j = i; + while (j < end && is_digit(text.charCodeAt(j))) j++; + if (j < end && text.charCodeAt(j) === 35 && j + 1 < end && is_alnum(text.charCodeAt(j + 1))) { + j++; + while (j < end && is_alnum(text.charCodeAt(j))) j++; + } + return j; +}; + +/** + * Skips a `'` or `"` quoted span (for balance scans), returning the index after + * the closing quote or the window end. Single quotes are literal; double quotes + * honor `\` escapes. + */ +const skip_bash_quote = (text: string, from: number, end: number, quote: number): number => { + let i = from + 1; + while (i < end) { + const c = text.charCodeAt(i); + if (c === 92 && quote === 34) i += 2; + else if (c === quote) return i + 1; + else i++; + } + return end; +}; + +/** + * Scans a `$'…'` ANSI-C string from the `$` at `i`, returning the exclusive + * end (with `\` escapes; unterminated extends to the window end). + */ +const scan_ansi_c = (text: string, i: number, end: number): number => { + let j = i + 2; + while (j < end) { + const c = text.charCodeAt(j); + if (c === 92) j += 2; + else if (c === 39) return j + 1; + else j++; + } + return end; +}; + +/** + * Finds the `)` closing a `$(` command substitution whose `(` sits before + * `from`, skipping strings and nested parens. Returns its index, or -1 when + * unbalanced within the window. + */ +const scan_cmdsub_end = (text: string, from: number, end: number): number => { + let depth = 0; + let j = from; + while (j < end) { + const c = text.charCodeAt(j); + if (c === 34 || c === 39) { + j = skip_bash_quote(text, j, end, c); + } else if (c === 40) { + depth++; + j++; + } else if (c === 41) { + if (depth === 0) return j; + depth--; + j++; + } else { + j++; + } + } + return -1; +}; + +/** + * Emits a `$`-variable at `i` (`${…}`, `$word`, or `$special`), returning the + * new position. A `$` with no valid expansion after it is plain text (returns + * `i + 1` with no token). + * + * @mutates `l` + */ +const scan_dollar_var = (l: Lexer, i: number, to: number): number => { + const {text} = l; + const c1 = text.charCodeAt(i + 1); + if (c1 === 123) { + // `${…}` — to the first `}` + const close = text.indexOf('}', i + 2); + const end = close === -1 || close >= to ? to : close + 1; + l.leaf(T_VARIABLE, i, end); + return end; + } + if (is_word_char(c1)) { + let j = i + 1; + while (j < to && is_word_char(text.charCodeAt(j))) j++; + l.leaf(T_VARIABLE, i, j); + return j; + } + if (SPECIAL_VAR.has(c1)) { + l.leaf(T_VARIABLE, i, i + 2); + return i + 2; + } + return i + 1; // bare `$` +}; + +/** + * Lexes a `$(…)` command substitution at `i` as a container, returning the new + * position. Interior lexes as ordinary bash; unterminated extends to `to`. + * + * @mutates `l` + */ +const lex_bash_cmdsub = (l: Lexer, i: number, to: number): number => { + const {text} = l; + l.open(T_COMMAND_SUBSTITUTION, i); + l.leaf(T_PUNCTUATION, i, i + 2); // `$(` + const close = scan_cmdsub_end(text, i + 2, to); + if (close === -1) { + lex_bash_window(l, i + 2, to); + l.close(to); + return to; + } + lex_bash_window(l, i + 2, close); + l.leaf(T_PUNCTUATION, close, close + 1); // `)` + l.close(close + 1); + return close + 1; +}; + +/** + * Lexes a `$((…))` arithmetic expansion at `i`, returning the new position. + * Distinct from command substitution: `$((`/`))` are punctuation and the + * interior lexes as ordinary bash. Unterminated extends to `to`. + * + * @mutates `l` + */ +const lex_bash_arith = (l: Lexer, i: number, to: number): number => { + const {text} = l; + let depth = 2; // the `((` + let j = i + 3; + while (j < to) { + const c = text.charCodeAt(j); + if (c === 34 || c === 39) { + j = skip_bash_quote(text, j, to, c); + } else if (c === 40) { + depth++; + j++; + } else if (c === 41) { + depth--; + if (depth === 0) break; + j++; + } else { + j++; + } + } + l.leaf(T_PUNCTUATION, i, i + 3); // `$((` + if (depth === 0) { + lex_bash_window(l, i + 3, j - 1); + l.leaf(T_PUNCTUATION, j - 1, j + 1); // `))` + return j + 1; + } + lex_bash_window(l, i + 3, to); // unterminated + return to; +}; + +/** + * Lexes a `"…"` double-quoted string at `i` as a container, returning the new + * position. `$`-expansions nest; unterminated extends to `to`. + * + * @mutates `l` + */ +const lex_bash_dquote = (l: Lexer, i: number, to: number): number => { + const {text} = l; + l.open(T_STRING, i); + let j = i + 1; + while (j < to) { + const c = text.charCodeAt(j); + if (c === 92) { + j += 2; + } else if (c === 34) { + j++; + break; + } else if (c === 36) { + const c1 = text.charCodeAt(j + 1); + if (c1 === 40) { + j = text.charCodeAt(j + 2) === 40 ? lex_bash_arith(l, j, to) : lex_bash_cmdsub(l, j, to); + } else { + j = scan_dollar_var(l, j, to); + } + } else { + j++; + } + } + if (j > to) j = to; + l.close(j); + return j; +}; + +interface HeredocClose { + // start of the delimiter word on its line (after leading blanks) + delim_start: number; + // exclusive end of the delimiter word + delim_end: number; + // start of the closing delimiter's line (end of the body region) + line_start: number; +} + +/** + * Finds the closing line of a heredoc with delimiter `delim`, scanning from + * `from`. A closing line is (optional leading blanks) + `delim` + (only blanks) + * before its newline. Returns the match, or `null` when unterminated. + */ +const find_heredoc_close = ( + text: string, + from: number, + end: number, + delim: string, +): HeredocClose | null => { + let j = from; + while (j < end) { + const line_start = j; + const le = line_end_of(text, j, end); + // the line's content excludes a trailing `\r` (CRLF) + const content_end = le > line_start && text.charCodeAt(le - 1) === 13 ? le - 1 : le; + const k = skip_blank(text, line_start, content_end); + if (text.startsWith(delim, k) && k + delim.length <= content_end) { + const rest = skip_blank(text, k + delim.length, content_end); + if (rest === content_end) { + return {delim_start: k, delim_end: k + delim.length, line_start}; + } + } + if (le >= end) break; + j = le + 1; + } + return null; +}; + +/** + * Emits `$`-expansions (`variable`/`command_substitution`) in an unquoted + * heredoc body `[from, to)`; other text stays plain. + * + * @mutates `l` + */ +const expand_heredoc_body = (l: Lexer, from: number, to: number): void => { + const {text} = l; + let j = from; + while (j < to) { + if (text.charCodeAt(j) === 36) { + const c1 = text.charCodeAt(j + 1); + if (c1 === 40) { + j = text.charCodeAt(j + 2) === 40 ? lex_bash_arith(l, j, to) : lex_bash_cmdsub(l, j, to); + } else { + j = scan_dollar_var(l, j, to); + } + } else { + j++; + } + } +}; + +interface PendingHeredoc { + delim: string; + quoted: boolean; +} + +/** + * Emits one heredoc body as a container (the opening delimiter was already + * emitted inline for the queued case), returning the position after it. + * + * @mutates `l` + */ +const emit_heredoc_body = ( + l: Lexer, + body_start: number, + to: number, + hd: PendingHeredoc, +): number => { + const {text} = l; + const close = find_heredoc_close(text, body_start, to, hd.delim); + l.open(T_HEREDOC, body_start); + const body_end = close ? close.line_start : to; + if (!hd.quoted) expand_heredoc_body(l, body_start, body_end); + if (close) { + l.leaf(T_HEREDOC_DELIMITER, close.delim_start, close.delim_end); + l.close(close.delim_end); + const nl = line_end_of(text, close.delim_end, to); + return nl >= to ? to : nl + 1; + } + l.close(to); + return to; +}; + +/** + * Drains queued heredocs whose bodies begin at `start` (just past a newline), + * consuming each body in order. Returns the position after the last one. + * + * @mutates `l` + */ +const drain_heredocs = ( + l: Lexer, + start: number, + to: number, + pending: Array, +): number => { + let j = start; + for (const hd of pending) { + if (j >= to) break; + j = emit_heredoc_body(l, j, to, hd); + } + pending.length = 0; + return j; +}; + +/** + * Scans a multi-char operator starting at `i` for the leading char `c` + * (one of `| & > !`); heredoc, `<<<`, `;;`, and `=` cases are handled by the + * caller. Returns its length. + */ +const bash_operator_len = (text: string, i: number, to: number, c: number): number => { + const c1 = i + 1 < to ? text.charCodeAt(i + 1) : 0; + switch (c) { + case 124: // | + return c1 === 124 ? 2 : 1; // || | + case 38: // & + if (c1 === 38) return 2; // && + if (c1 === 62) return text.charCodeAt(i + 2) === 62 ? 3 : 2; // &>> &> + return 1; // & + case 62: // > + return c1 === 62 ? 2 : 1; // >> > + default: // 33 `!` + return c1 === 61 ? 2 : 1; // != ! + } +}; + +/** + * The core window lexer. Maintains a per-window queue of pending heredocs + * (redirected on the current line, bodies consumed at the next newline). + */ +const lex_bash_window = (l: Lexer, from: number, to: number): void => { + const {text} = l; + const pending: Array = []; + let i = from; + let prev_function_kw = false; + + while (i < to) { + const c = text.charCodeAt(i); + + if (c === 10) { + i = pending.length > 0 ? drain_heredocs(l, i + 1, to, pending) : i + 1; + continue; + } + if (is_space(c)) { + i++; + continue; + } + + const fn = prev_function_kw; + prev_function_kw = false; + + // shebang / comments + if (c === 35) { + if (i === 0 && text.charCodeAt(1) === 33) { + const le = scan_to_line_end(text, i, to); + l.leaf(T_SHEBANG, i, le); + i = le; + continue; + } + if (i === 0 || is_space(text.charCodeAt(i - 1))) { + const le = scan_to_line_end(text, i, to); + l.leaf(T_COMMENT, i, le); + i = le; + continue; + } + i++; + continue; + } + + // `$` expansions + if (c === 36) { + const c1 = text.charCodeAt(i + 1); + if (c1 === 40) { + i = text.charCodeAt(i + 2) === 40 ? lex_bash_arith(l, i, to) : lex_bash_cmdsub(l, i, to); + continue; + } + if (c1 === 39) { + const e = scan_ansi_c(text, i, to); + l.leaf(T_STRING, i, e); + i = e; + continue; + } + i = scan_dollar_var(l, i, to); + continue; + } + + // strings + if (c === 34) { + i = lex_bash_dquote(l, i, to); + continue; + } + if (c === 39) { + const close = text.indexOf("'", i + 1); + const e = close === -1 || close >= to ? to : close + 1; + l.leaf(T_STRING, i, e); + i = e; + continue; + } + + // numbers and file descriptors + if (is_digit(c)) { + const next = text.charCodeAt(i + 1); + if (next === 62 || next === 60) { + // `\b\d(?=>>?|<)` — a single-digit file descriptor + l.leaf(T_FILE_DESCRIPTOR, i, i + 1); + i++; + continue; + } + const num_end = scan_bash_number(text, i, to); + if (!is_word_char(text.charCodeAt(num_end))) { + l.leaf(T_NUMBER, i, num_end); + i = num_end; + continue; + } + // digit-led but part of a larger word — fall through to word handling + } + + // words: function defs, keywords, builtins, booleans, else plain + if (is_word_char(c)) { + let wend = i + 1; + while (wend < to && is_word_char(text.charCodeAt(wend))) wend++; + if (fn) { + l.leaf(T_FUNCTION, i, wend); + i = wend; + continue; + } + const a = skip_space(text, wend, to); + if (text.charCodeAt(a) === 40 && text.charCodeAt(skip_space(text, a + 1, to)) === 41) { + l.leaf(T_FUNCTION, i, wend); // `name()` definition + i = wend; + continue; + } + const word = text.slice(i, wend); + const kind = WORDS.get(word); + if (kind === K_KEYWORD) { + l.leaf(T_KEYWORD, i, wend); + if (word === 'function') prev_function_kw = true; + } else if (kind === K_BUILTIN) { + l.leaf(T_BUILTIN, i, wend); + } else if (kind === K_BOOLEAN) { + l.leaf(T_BOOLEAN, i, wend); + } + i = wend; + continue; + } + + // `<` — here-string, heredoc, or `<`/`<<` operator + if (c === 60) { + if (text.charCodeAt(i + 1) === 60) { + if (text.charCodeAt(i + 2) === 60) { + l.leaf(T_OPERATOR, i, i + 3); // `<<<` + i += 3; + continue; + } + const consumed = lex_bash_heredoc_start(l, i, to, pending); + if (consumed !== -1) { + i = consumed; + continue; + } + l.leaf(T_OPERATOR, i, i + 2); // `<<` with no delimiter + i += 2; + continue; + } + l.leaf(T_OPERATOR, i, i + 1); // `<` + i++; + continue; + } + + // `&d` file descriptor before the `&` operator forms + if ( + c === 38 && + is_digit(text.charCodeAt(i + 1)) && + !is_word_char(text.charCodeAt(i + 2)) && + (i === 0 || !is_word_char(text.charCodeAt(i - 1))) + ) { + l.leaf(T_FILE_DESCRIPTOR, i, i + 2); + i += 2; + continue; + } + + // `;;` operator vs `;` punctuation + if (c === 59) { + if (text.charCodeAt(i + 1) === 59) { + l.leaf(T_OPERATOR, i, i + 2); + i += 2; + } else { + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + } + continue; + } + + // `=` — plain, except `==` / `=~` + if (c === 61) { + const c1 = text.charCodeAt(i + 1); + if (c1 === 126 || c1 === 61) { + l.leaf(T_OPERATOR, i, i + 2); + i += 2; + } else { + i++; + } + continue; + } + + // operators + if (c === 124 || c === 38 || c === 62 || c === 33) { + const len = bash_operator_len(text, i, to, c); + l.leaf(T_OPERATOR, i, i + len); + i += len; + continue; + } + + // punctuation + if (c === 123 || c === 125 || c === 91 || c === 93 || c === 40 || c === 41 || c === 44) { + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + continue; + } + + i++; // anything else is plain text + } + + // unterminated trailing heredocs (opener consumed, body ran to window end) + if (pending.length > 0) drain_heredocs(l, i, to, pending); + l.pos = to; +}; + +/** + * Handles a `<<`/`<<-` heredoc redirect at `i`. Emits the opening delimiter and + * either consumes the whole heredoc inline (when it is the sole redirect at the + * end of the line) or queues it for draining at the next newline. Returns the + * new position, or -1 when no delimiter follows (the caller emits `<<` as an + * operator). + * + * @mutates `l` + */ +const lex_bash_heredoc_start = ( + l: Lexer, + i: number, + to: number, + pending: Array, +): number => { + const {text} = l; + let k = i + 2; + if (text.charCodeAt(k) === 45) k++; // `<<-` + k = skip_blank(text, k, to); + let quote = 0; + if (text.charCodeAt(k) === 39 || text.charCodeAt(k) === 34) { + quote = text.charCodeAt(k); + k++; + } + const dstart = k; + while (k < to && is_word_char(text.charCodeAt(k))) k++; + if (k === dstart) return -1; // no delimiter word + const delim = text.slice(dstart, k); + let delim_token_end = k; + if (quote !== 0 && text.charCodeAt(k) === quote) delim_token_end = k + 1; + + const after = skip_blank(text, delim_token_end, to); + const contiguous = after >= to || text.charCodeAt(after) === 10; + const quoted = quote !== 0; + + // non-contiguous, or another heredoc already queued: defer the body so the + // queued order (first redirect → first body) is preserved + if (!contiguous || pending.length > 0) { + l.leaf(T_HEREDOC_DELIMITER, i, delim_token_end); + pending.push({delim, quoted}); + return delim_token_end; + } + + // contiguous single heredoc — one container from opener to closing delimiter + if (after >= to) { + l.leaf(T_HEREDOC_DELIMITER, i, delim_token_end); + return delim_token_end; + } + const body_start = after + 1; // past the newline + const close = find_heredoc_close(text, body_start, to, delim); + l.open(T_HEREDOC, i); + l.leaf(T_HEREDOC_DELIMITER, i, delim_token_end); + const body_end = close ? close.line_start : to; + if (!quoted) expand_heredoc_body(l, body_start, body_end); + if (close) { + l.leaf(T_HEREDOC_DELIMITER, close.delim_start, close.delim_end); + l.close(close.delim_end); + return close.delim_end; + } + l.close(to); + return to; +}; + +const lex_bash = (l: Lexer): void => { + lex_bash_window(l, l.pos, l.end); +}; + +/** + * The Bash language registration for the lexer engine. + */ +export const lexer_bash: SyntaxLang = {id: 'bash', aliases: ['sh', 'shell'], lex: lex_bash}; diff --git a/src/lib/lexer_css.ts b/src/lib/lexer_css.ts new file mode 100644 index 00000000..35490082 --- /dev/null +++ b/src/lib/lexer_css.ts @@ -0,0 +1,340 @@ +import {is_space, token_type, type Lexer, type SyntaxLang} from './lexer.ts'; + +/** + * Hand-written CSS lexer. + * + * Token vocabulary matches the retired regex grammar: `comment`, `atrule` + * (a container wrapping `rule` + prelude), `rule`, `selector`, `string`, + * `property`, `important`, `function`, `url` (a container), `keyword` + * (`and`/`not`/`only`/`or` inside at-rule preludes), and `punctuation`. + * + * Structure is decided by a brace-context lookahead rather than the old + * lookahead regexes: from each item, the first top-level `{`, `;`, or `}` + * (skipping strings, comments, `()`, and `[]`) says whether the item is a + * qualified rule (its prelude is a `selector`) or a declaration + * (`property : value`). Because that decision is purely local, the flat main + * loop nests to any depth — native CSS nesting works for free (the agreed + * fidelity extra), where the regex model could not express it. + * + * Values (declaration right-hand sides and at-rule preludes) reproduce the old + * grammar's sparse output: only strings, `url()`, functions, `!important`, and + * `property`-before-colon are tokenized; numbers, colors, units, and bare + * identifiers stay plain text. + * + * Resilience: unterminated strings extend to end of line; unterminated + * comments and blocks extend to end of window. + */ + +const T_COMMENT = token_type('comment'); +const T_ATRULE = token_type('atrule'); +const T_RULE = token_type('rule'); +const T_SELECTOR = token_type('selector'); +const T_STRING = token_type('string'); +const T_STRING_URL = token_type('string', 'url'); +const T_PROPERTY = token_type('property'); +const T_IMPORTANT = token_type('important'); +const T_FUNCTION = token_type('function'); +const T_PUNCTUATION = token_type('punctuation'); +const T_URL = token_type('url'); +const T_KEYWORD = token_type('keyword'); + +// at-rule prelude keywords (`@media screen and (...)`) +const ATRULE_KEYWORDS: Set = new Set(['and', 'not', 'only', 'or']); + +// a css identifier char: letters, digits, `-`, `_`, and non-ASCII +const is_css_ident = (c: number): boolean => + (c >= 97 && c <= 122) || // a-z + (c >= 65 && c <= 90) || // A-Z + (c >= 48 && c <= 57) || // 0-9 + c === 45 || // - + c === 95 || // _ + c >= 0xa0; + +// a css identifier start: no leading digit +const is_css_ident_start = (c: number): boolean => + (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || c === 45 || c === 95 || c >= 0xa0; + +/** + * Scans a `"` or `'` string from `i`, returning the exclusive end. + * Handles `\` escapes (including escaped newlines); an unterminated string + * stops at the newline (matching the old grammar's `[^"\\\r\n]` class). + */ +const scan_css_string = (text: string, from: number, end: number, quote: number): number => { + let i = from + 1; + while (i < end) { + const c = text.charCodeAt(i); + if (c === 92) { + i += 2; // escape + } else if (c === quote) { + return i + 1; + } else if (c === 10 || c === 13) { + return i; // unterminated + } else { + i++; + } + } + return end; +}; + +/** + * Scans a block comment from the slash-star at `i`, returning the exclusive + * end. Unterminated comments extend to the window end. + */ +const scan_css_comment = (text: string, i: number, end: number): number => { + const close = text.indexOf('*/', i + 2); + return close === -1 || close + 2 > end ? end : close + 2; +}; + +/** + * From `i`, finds the first top-level `{`, `;`, or `}`, skipping strings, + * comments, and balanced `()`/`[]`. Returns its index, or `end` when none is + * found. This is the selector-vs-declaration discriminator. + */ +const scan_to_terminator = (text: string, i: number, end: number): number => { + let paren = 0; + let bracket = 0; + let j = i; + while (j < end) { + const c = text.charCodeAt(j); + if (c === 34 || c === 39) { + j = scan_css_string(text, j, end, c); + } else if (c === 47 && text.charCodeAt(j + 1) === 42) { + j = scan_css_comment(text, j, end); + } else if (c === 40) { + paren++; + j++; + } else if (c === 41) { + if (paren > 0) paren--; + j++; + } else if (c === 91) { + bracket++; + j++; + } else if (c === 93) { + if (bracket > 0) bracket--; + j++; + } else if (paren === 0 && bracket === 0 && (c === 123 || c === 59 || c === 125)) { + return j; + } else { + j++; + } + } + return end; +}; + +// trims trailing whitespace from a `[from, to)` span, returning the new `to` +const trim_end = (text: string, from: number, to: number): number => { + let end = to; + while (end > from && is_space(text.charCodeAt(end - 1))) end--; + return end; +}; + +/** + * Tokenizes a `[from, to)` value region — a declaration right-hand side or an + * at-rule prelude interior. Emits strings, `url()`, functions, `!important`, + * `property`-before-colon, punctuation, and (when `is_atrule`) + * `and`/`not`/`only`/`or` keywords; everything else is plain text. + */ +const lex_css_value = (l: Lexer, from: number, to: number, is_atrule: boolean): void => { + const {text} = l; + let i = from; + while (i < to) { + const c = text.charCodeAt(i); + if (is_space(c)) { + i++; + continue; + } + if (c === 47 && text.charCodeAt(i + 1) === 42) { + const close = scan_css_comment(text, i, to); + l.leaf(T_COMMENT, i, close); + i = close; + continue; + } + if (c === 34 || c === 39) { + const str_end = scan_css_string(text, i, to, c); + l.leaf(T_STRING, i, str_end); + i = str_end; + continue; + } + if (c === 33) { + // `!important` (case-insensitive), with a word boundary after + const word_end = i + 10; // `!important` + if ( + word_end <= to && + matches_ci(text, i + 1, 'important') && + !is_css_ident(text.charCodeAt(word_end)) + ) { + l.leaf(T_IMPORTANT, i, word_end); + i = word_end; + continue; + } + i++; + continue; + } + if (is_css_ident_start(c)) { + let ident_end = i + 1; + while (ident_end < to && is_css_ident(text.charCodeAt(ident_end))) ident_end++; + // `url(...)` — a container, not a plain function call + if (ident_end - i === 3 && matches_ci(text, i, 'url') && text.charCodeAt(ident_end) === 40) { + i = lex_css_url(l, i, ident_end, to); + continue; + } + const next = ident_end; + const nc = text.charCodeAt(next); + if (nc === 40) { + l.leaf(T_FUNCTION, i, ident_end); + i = ident_end; + continue; + } + if (nc === 58) { + l.leaf(T_PROPERTY, i, ident_end); + i = ident_end; + continue; + } + if (is_atrule && ATRULE_KEYWORDS.has(text.slice(i, ident_end).toLowerCase())) { + l.leaf(T_KEYWORD, i, ident_end); + i = ident_end; + continue; + } + i = ident_end; // plain identifier (color name, unit-bearing token, …) + continue; + } + if (c === 40 || c === 41 || c === 91 || c === 93 || c === 44 || c === 58 || c === 59) { + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + continue; + } + i++; // numbers, `#hex`, `%`, `.`, `-` before digits, … stay plain + } +}; + +/** + * Lexes a `url(...)` construct as a container. `i` is the `u`, `word_end` the + * `(`. Returns the position after the closing `)` (or the region end when + * unterminated). + */ +const lex_css_url = (l: Lexer, i: number, word_end: number, to: number): number => { + const {text} = l; + l.open(T_URL, i); + l.leaf(T_FUNCTION, i, word_end); + l.leaf(T_PUNCTUATION, word_end, word_end + 1); // `(` + let j = word_end + 1; + while (j < to && is_space(text.charCodeAt(j))) j++; + const c = text.charCodeAt(j); + if (c === 34 || c === 39) { + const str_end = scan_css_string(text, j, to, c); + l.leaf(T_STRING_URL, j, str_end); + j = str_end; + } + // raw (unquoted) content and trailing space up to `)` stay plain + const close = text.indexOf(')', j); + if (close === -1 || close >= to) { + l.close(to); + return to; + } + l.leaf(T_PUNCTUATION, close, close + 1); + l.close(close + 1); + return close + 1; +}; + +// case-insensitive ASCII match of `word` at `text[from..]` +const matches_ci = (text: string, from: number, word: string): boolean => { + for (let k = 0; k < word.length; k++) { + if ((text.charCodeAt(from + k) | 0x20) !== word.charCodeAt(k)) return false; + } + return true; +}; + +/** + * Lexes a declaration `[from, to)` — `property : value` — falling back to a + * bare value when there is no top-level `property:`. + */ +const lex_css_declaration = (l: Lexer, from: number, to: number): void => { + const {text} = l; + let i = from; + while (i < to && is_space(text.charCodeAt(i))) i++; + if (i < to && is_css_ident_start(text.charCodeAt(i))) { + let name_end = i + 1; + while (name_end < to && is_css_ident(text.charCodeAt(name_end))) name_end++; + let after = name_end; + while (after < to && is_space(text.charCodeAt(after))) after++; + if (after < to && text.charCodeAt(after) === 58) { + l.leaf(T_PROPERTY, i, name_end); + l.leaf(T_PUNCTUATION, after, after + 1); // `:` + lex_css_value(l, after + 1, to, false); + return; + } + } + lex_css_value(l, from, to, false); +}; + +/** + * Lexes an at-rule from the `@` at `i`, returning the new position. Emits an + * `atrule` container (`rule` + prelude), then the terminating `{` or `;`. + */ +const lex_css_atrule = (l: Lexer, i: number, end: number): number => { + const {text} = l; + let rule_end = i + 1; + while (rule_end < end && is_css_ident(text.charCodeAt(rule_end))) rule_end++; + const term = scan_to_terminator(text, rule_end, end); + const prelude_end = trim_end(text, i, term); + l.open(T_ATRULE, i); + l.leaf(T_RULE, i, rule_end); + lex_css_value(l, rule_end, prelude_end, true); + l.close(prelude_end); + const tc = text.charCodeAt(term); + if (tc === 123 || tc === 59) { + l.leaf(T_PUNCTUATION, term, term + 1); + return term + 1; + } + return term; // `}` handled by the caller, or region end +}; + +const lex_css = (l: Lexer): void => { + const {text, end} = l; + let i = l.pos; + while (i < end) { + const c = text.charCodeAt(i); + if (is_space(c)) { + i++; + continue; + } + if (c === 47 && text.charCodeAt(i + 1) === 42) { + const close = scan_css_comment(text, i, end); + l.leaf(T_COMMENT, i, close); + i = close; + continue; + } + if (c === 125 || c === 123 || c === 59) { + // `}` closes a block; stray `{`/`;` are emitted as punctuation too + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + continue; + } + if (c === 64) { + i = lex_css_atrule(l, i, end); + continue; + } + // a qualified rule (selector) or a declaration — decided by lookahead + const term = scan_to_terminator(text, i, end); + if (text.charCodeAt(term) === 123) { + const sel_end = trim_end(text, i, term); + l.leaf(T_SELECTOR, i, sel_end); + l.leaf(T_PUNCTUATION, term, term + 1); // `{` + i = term + 1; + } else { + lex_css_declaration(l, i, term); + if (text.charCodeAt(term) === 59) { + l.leaf(T_PUNCTUATION, term, term + 1); // `;` + i = term + 1; + } else { + i = term; // `}` handled next iteration, or region end + } + } + } + l.pos = end; +}; + +/** + * The CSS language registration for the lexer engine. + */ +export const lexer_css: SyntaxLang = {id: 'css', lex: lex_css}; diff --git a/src/lib/lexer_json.ts b/src/lib/lexer_json.ts new file mode 100644 index 00000000..a34bb411 --- /dev/null +++ b/src/lib/lexer_json.ts @@ -0,0 +1,156 @@ +import { + is_digit, + is_ident_start, + is_space, + scan_ident, + scan_to_line_end, + token_type, + type Lexer, + type SyntaxLang, +} from './lexer.ts'; + +/** + * Hand-written JSON lexer (accepts JSONC — line and block comments). + * + * Token vocabulary matches the retired regex grammar: `property` (a string + * key), `string`, `comment`, `number`, `punctuation`, `operator` (`:`), + * `boolean`, and `null` (aliased to `keyword`). + * + * Resilience: unterminated strings extend to end of line; unterminated block + * comments extend to end of window. + */ + +const T_PROPERTY = token_type('property'); +const T_STRING = token_type('string'); +const T_COMMENT = token_type('comment'); +const T_NUMBER = token_type('number'); +const T_PUNCTUATION = token_type('punctuation'); +const T_OPERATOR = token_type('operator'); +const T_BOOLEAN = token_type('boolean'); +const T_NULL = token_type('null', 'keyword'); + +/** + * Scans a `"` string from `i`, returning the exclusive end index. + * Stops before an unescaped newline (unterminated) or at end of window. + */ +const scan_json_string = (text: string, from: number, end: number): number => { + let i = from + 1; + while (i < end) { + const c = text.charCodeAt(i); + if (c === 92) { + i += 2; // escape — skip escaped char + } else if (c === 34) { + return i + 1; + } else if (c === 10 || c === 13) { + return i; // unterminated: stop at the line boundary + } else { + i++; + } + } + return end; +}; + +/** + * Scans a number from `i` (at `-` or a digit), returning the exclusive end. + */ +const scan_json_number = (text: string, from: number, end: number): number => { + let i = from; + if (text.charCodeAt(i) === 45) i++; // - + while (i < end && is_digit(text.charCodeAt(i))) i++; + if (i < end && text.charCodeAt(i) === 46 && is_digit(text.charCodeAt(i + 1))) { + i += 2; + while (i < end && is_digit(text.charCodeAt(i))) i++; + } + const c = i < end ? text.charCodeAt(i) : 0; + if (c === 101 || c === 69) { + // e/E exponent + let j = i + 1; + const sign = j < end ? text.charCodeAt(j) : 0; + if (sign === 43 || sign === 45) j++; + if (j < end && is_digit(text.charCodeAt(j))) { + j++; + while (j < end && is_digit(text.charCodeAt(j))) j++; + return j; + } + } + return i; +}; + +const lex_json = (l: Lexer): void => { + const {text, end} = l; + let i = l.pos; + while (i < end) { + const c = text.charCodeAt(i); + if (is_space(c)) { + i++; + continue; + } + if (c === 34) { + // `"` — string; a key when followed by `:` + const str_end = scan_json_string(text, i, end); + let j = str_end; + while (j < end && is_space(text.charCodeAt(j))) j++; + l.leaf(j < end && text.charCodeAt(j) === 58 ? T_PROPERTY : T_STRING, i, str_end); + i = str_end; + continue; + } + if (c === 47) { + // `/` — jsonc comments + const c2 = i + 1 < end ? text.charCodeAt(i + 1) : 0; + if (c2 === 47) { + const line_end = scan_to_line_end(text, i, end); + l.leaf(T_COMMENT, i, line_end); + i = line_end; + continue; + } + if (c2 === 42) { + const close = text.indexOf('*/', i + 2); + const comment_end = close === -1 || close + 2 > end ? end : close + 2; + l.leaf(T_COMMENT, i, comment_end); + i = comment_end; + continue; + } + i++; + continue; + } + if (c === 45 || is_digit(c)) { + if (c === 45 && !is_digit(i + 1 < end ? text.charCodeAt(i + 1) : 0)) { + i++; // lone `-` is plain text + continue; + } + const num_end = scan_json_number(text, i, end); + l.leaf(T_NUMBER, i, num_end); + i = num_end; + continue; + } + if (c === 123 || c === 125 || c === 91 || c === 93 || c === 44) { + // { } [ ] , + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + continue; + } + if (c === 58) { + l.leaf(T_OPERATOR, i, i + 1); + i++; + continue; + } + if (is_ident_start(c)) { + const ident_end = scan_ident(text, i, end); + const word = text.slice(i, ident_end); + if (word === 'true' || word === 'false') { + l.leaf(T_BOOLEAN, i, ident_end); + } else if (word === 'null') { + l.leaf(T_NULL, i, ident_end); + } + i = ident_end; + continue; + } + i++; + } + l.pos = i; +}; + +/** + * The JSON language registration for the lexer engine. + */ +export const lexer_json: SyntaxLang = {id: 'json', lex: lex_json}; diff --git a/src/lib/lexer_ts.ts b/src/lib/lexer_ts.ts new file mode 100644 index 00000000..434ff486 --- /dev/null +++ b/src/lib/lexer_ts.ts @@ -0,0 +1,988 @@ +import { + is_digit, + is_ident, + is_ident_start, + is_space, + scan_ident, + scan_to_line_end, + skip_space, + token_type, + type Lexer, + type SyntaxLang, +} from './lexer.ts'; + +/** + * Hand-written TypeScript/JavaScript lexer. + * + * Token vocabulary matches the retired regex grammars (`grammar_js.ts` + + * `grammar_ts.ts`) with the agreed fidelity fixes: previous-token tracking for + * regex-literal vs division, unlimited template/interpolation nesting, and + * position-major matching (strings win over comments that start inside them). + * + * Resilience: unterminated strings extend to end of line; unterminated + * templates, block comments, and interpolations extend to end of window. + */ + +const T_COMMENT = token_type('comment'); +const T_HASHBANG = token_type('hashbang', 'comment'); +const T_STRING = token_type('string'); +const T_STRING_PROPERTY = token_type('string_property', 'property'); +const T_TEMPLATE_STRING = token_type('template_string'); +const T_TEMPLATE_PUNCTUATION = token_type('template_punctuation', 'string'); +const T_INTERPOLATION = token_type('interpolation'); +const T_INTERPOLATION_PUNCTUATION = token_type('interpolation_punctuation', 'punctuation'); +const T_REGEX = token_type('regex'); +const T_REGEX_DELIMITER = token_type('regex_delimiter'); +const T_REGEX_SOURCE = token_type('regex_source', 'lang_regex'); +const T_REGEX_FLAGS = token_type('regex_flags'); +const T_KEYWORD = token_type('keyword'); +const T_SPECIAL_KEYWORD = token_type('special_keyword'); +const T_IMPORT_TYPE_KEYWORD = token_type('import_type_keyword', 'special_keyword'); +const T_CLASS_NAME = token_type('class_name'); +const T_TYPE_NAME = token_type('type_name', 'class_name'); +const T_TYPE_ASSERTION = token_type('type_assertion', 'class_name'); +const T_FUNCTION = token_type('function'); +const T_FUNCTION_VARIABLE = token_type('function_variable', 'function'); +const T_GENERIC_FUNCTION = token_type('generic_function'); +const T_GENERIC = token_type('generic', 'class_name'); +const T_BUILTIN = token_type('builtin'); +const T_BOOLEAN = token_type('boolean'); +const T_NUMBER = token_type('number'); +const T_OPERATOR = token_type('operator'); +const T_PUNCTUATION = token_type('punctuation'); +const T_CONSTANT = token_type('constant'); +const T_CAPITALIZED = token_type('capitalized_identifier', 'class_name'); +const T_DECORATOR = token_type('decorator'); +const T_AT = token_type('at', 'operator'); +const T_DECORATOR_NAME = token_type('function', 'decorator_name'); +const T_TYPE_ANNOTATION = token_type('type_annotation'); +const T_COLON = token_type(':'); +const T_TYPE = token_type('type'); + +// word classification kinds +const K_KEYWORD = 1; // unconditional keyword +const K_SPECIAL = 2; // unconditional special_keyword +const K_TS = 3; // ts-only unconditional keyword +const K_ASYNC = 4; // keyword before `function`/`*`/`(`/ident +const K_GET_SET = 5; // keyword before ident/`#`/`[` +const K_ASSERT = 6; // keyword before `{` +const K_TYPE_WORD = 7; // `type` — import_type_keyword or keyword before ident/`{`/`*` +const K_TS_COND = 8; // ts keyword before ident/`{` +const K_BOOLEAN = 9; +const K_NUMBER_WORD = 10; // NaN/Infinity + +const WORDS: Map = new Map(); +const add_words = (kind: number, words: string): void => { + for (const word of words.split(' ')) WORDS.set(word, kind); +}; +add_words( + K_KEYWORD, + 'class const debugger delete enum extends function implements in instanceof interface let ' + + 'new null of package private protected public static super this typeof undefined var void with', +); +add_words( + K_SPECIAL, + 'as await break case catch continue default do else export finally for from if import ' + + 'return switch throw try while yield', +); +add_words(K_TS, 'abstract declare is keyof readonly require satisfies'); +add_words(K_TS_COND, 'asserts infer module namespace'); +add_words(K_ASYNC, 'async'); +add_words(K_GET_SET, 'get set'); +add_words(K_ASSERT, 'assert'); +add_words(K_TYPE_WORD, 'type'); +add_words(K_BOOLEAN, 'true false'); +add_words(K_NUMBER_WORD, 'NaN Infinity'); + +// keywords that put the lexer in a class-name context for the next identifier +const CLASS_CTX_WORDS: Set = new Set([ + 'class', + 'extends', + 'implements', + 'instanceof', + 'interface', + 'new', + 'type', +]); +// keywords after which the next identifier is a type assertion +const AS_WORDS: Set = new Set(['as', 'satisfies']); +const IMPORT_WORDS: Set = new Set(['import', 'export']); +// value-like keywords — division follows, not a regex literal +const VALUE_WORDS: Set = new Set(['this', 'super', 'null', 'undefined']); + +// lowercase-only builtins are reachable — capitalized ones are claimed by +// `capitalized_identifier` first, matching the old grammar's priority order +const BUILTIN_WORDS: Set = new Set([ + 'Array', + 'Function', + 'Promise', + 'any', + 'boolean', + 'console', + 'never', + 'number', + 'string', + 'symbol', + 'unknown', +]); + +// previous-significant-token categories, for regex-vs-division and contexts +const P_NONE = 0; // start / after operator, `{`, `(`, `[`, `,`, `;` — regex allowed +const P_VALUE = 1; // after a value — `/` is division +const P_DOT = 2; // after `.` member access — next word is a property, not a keyword + +// bounded lookahead for heuristic scans (generics, annotations, arrow params) +const MAX_SCAN = 600; + +const is_upper = (c: number): boolean => c >= 65 && c <= 90; + +/** + * Scans a `'` or `"` string from `i`, returning the exclusive end. + * Handles escapes and line continuations; unterminated stops at the newline. + */ +const scan_ts_string = (text: string, from: number, end: number, quote: number): number => { + let i = from + 1; + while (i < end) { + const c = text.charCodeAt(i); + if (c === 92) { + // escape; a `\` before a newline continues the string + i += text.charCodeAt(i + 1) === 13 && text.charCodeAt(i + 2) === 10 ? 3 : 2; + } else if (c === quote) { + return i + 1; + } else if (c === 10 || c === 13) { + return i; + } else { + i++; + } + } + return end; +}; + +/** + * Scans a numeric literal from `i` (at a digit, or `.` before a digit), + * returning the exclusive end. Handles hex/binary/octal, `_` separators, + * exponents, and bigint `n` suffixes. + */ +const scan_ts_number = (text: string, i: number, end: number): number => { + const scan_digits = (from: number, is_wanted: (c: number) => boolean): number => { + let j = from; + while (j < end) { + const c = text.charCodeAt(j); + if (is_wanted(c) || (c === 95 && is_wanted(text.charCodeAt(j + 1)))) j++; + else break; + } + return j; + }; + const is_hex = (c: number): boolean => + (c >= 48 && c <= 57) || (c >= 97 && c <= 102) || (c >= 65 && c <= 70); + const is_binary = (c: number): boolean => c === 48 || c === 49; + const is_octal = (c: number): boolean => c >= 48 && c <= 55; + + if (text.charCodeAt(i) === 48) { + const c2 = text.charCodeAt(i + 1); + if (c2 === 120 || c2 === 88) { + let j = scan_digits(i + 2, is_hex); + if (text.charCodeAt(j) === 110) j++; // n + return j; + } + if (c2 === 98 || c2 === 66) { + let j = scan_digits(i + 2, is_binary); + if (text.charCodeAt(j) === 110) j++; + return j; + } + if (c2 === 111 || c2 === 79) { + let j = scan_digits(i + 2, is_octal); + if (text.charCodeAt(j) === 110) j++; + return j; + } + } + let j = scan_digits(i, is_digit); + let is_integer = true; + if (text.charCodeAt(j) === 46 && is_digit(text.charCodeAt(j + 1))) { + is_integer = false; + j = scan_digits(j + 1, is_digit); + } + const e = text.charCodeAt(j); + if (e === 101 || e === 69) { + let k = j + 1; + const sign = text.charCodeAt(k); + if (sign === 43 || sign === 45) k++; + if (is_digit(text.charCodeAt(k))) { + j = scan_digits(k, is_digit); + is_integer = false; + } + } + if (is_integer && text.charCodeAt(j) === 110) j++; // bigint + return j; +}; + +/** + * Skips a quoted span (used inside balanced scans), returning the index after + * the closing quote, or after the span's line for unterminated strings. + */ +const skip_quoted = (text: string, from: number, end: number, quote: number): number => { + let i = from + 1; + while (i < end) { + const c = text.charCodeAt(i); + if (c === 92) i += 2; + else if (c === quote) return i + 1; + else if ((c === 10 || c === 13) && quote !== 96) return i; + else i++; + } + return end; +}; + +/** + * Finds the matching `}` for the `{` at `i`, skipping strings, templates, and + * comments. Returns -1 when unbalanced within the window. + */ +const scan_balanced_braces = (text: string, i: number, end: number): number => { + let depth = 0; + let j = i; + while (j < end) { + const c = text.charCodeAt(j); + if (c === 123) { + depth++; + j++; + } else if (c === 125) { + depth--; + if (depth === 0) return j; + j++; + } else if (c === 34 || c === 39 || c === 96) { + j = skip_quoted(text, j, end, c); + } else if (c === 47) { + const c2 = text.charCodeAt(j + 1); + if (c2 === 47) { + j = scan_to_line_end(text, j, end); + } else if (c2 === 42) { + const close = text.indexOf('*/', j + 2); + j = close === -1 || close + 2 > end ? end : close + 2; + } else { + j++; + } + } else { + j++; + } + } + return -1; +}; + +/** + * Finds the matching `>` for the `<` at `i` (generic argument lists), skipping + * strings. Bounded by `MAX_SCAN`; returns -1 when unbalanced. + */ +const scan_balanced_angle = (text: string, i: number, end: number): number => { + const limit = i + MAX_SCAN < end ? i + MAX_SCAN : end; + let depth = 0; + let j = i; + while (j < limit) { + const c = text.charCodeAt(j); + if (c === 60) { + depth++; + j++; + } else if (c === 62) { + depth--; + if (depth === 0) return j; + j++; + } else if (c === 34 || c === 39 || c === 96) { + j = skip_quoted(text, j, end, c); + } else if (c === 59) { + return -1; // `;` never appears in a generic argument list + } else { + j++; + } + } + return -1; +}; + +/** + * Finds the matching `)` for the `(` at `i`, skipping strings and nested + * parens. Bounded by `MAX_SCAN`; returns -1 when unbalanced. + */ +const scan_balanced_parens = (text: string, i: number, end: number): number => { + const limit = i + MAX_SCAN < end ? i + MAX_SCAN : end; + let depth = 0; + let j = i; + while (j < limit) { + const c = text.charCodeAt(j); + if (c === 40) { + depth++; + j++; + } else if (c === 41) { + depth--; + if (depth === 0) return j; + j++; + } else if (c === 34 || c === 39 || c === 96) { + j = skip_quoted(text, j, end, c); + } else { + j++; + } + } + return -1; +}; + +/** + * Detects whether the identifier ending at `ident_end` is a function-valued + * variable: `ident` followed by `=` or `:`, then optional `async`, then a + * `function` keyword, `(params) =>`, or `param =>`. + */ +const is_function_variable = (text: string, ident_end: number, end: number): boolean => { + let j = skip_space(text, ident_end, end); + const c = text.charCodeAt(j); + if (c === 61) { + // `=` — but not `==`/`=>`/`===` + const c2 = text.charCodeAt(j + 1); + if (c2 === 61 || c2 === 62) return false; + } else if (c !== 58) { + return false; + } + j = skip_space(text, j + 1, end); + // optional `async` + if (text.startsWith('async', j)) { + const after = j + 5; + if (!is_ident(text.charCodeAt(after))) j = skip_space(text, after, end); + } + if (text.startsWith('function', j) && !is_ident(text.charCodeAt(j + 8))) return true; + const c3 = text.charCodeAt(j); + if (c3 === 40) { + // `(params)` then optional `: type` then `=>` + const close = scan_balanced_parens(text, j, end); + if (close === -1) return false; + let k = skip_space(text, close + 1, end); + if (text.charCodeAt(k) === 58) { + // return type annotation — scan to `=>` before a terminator + const limit = k + MAX_SCAN < end ? k + MAX_SCAN : end; + k++; + while (k < limit) { + const c4 = text.charCodeAt(k); + if (c4 === 61 && text.charCodeAt(k + 1) === 62) break; + if (c4 === 59 || c4 === 61) return false; + k++; + } + } + return text.charCodeAt(k) === 61 && text.charCodeAt(k + 1) === 62; + } + if (is_ident_start(c3)) { + // `param =>` + const param_end = scan_ident(text, j, end); + const k = skip_space(text, param_end, end); + return text.charCodeAt(k) === 61 && text.charCodeAt(k + 1) === 62; + } + return false; +}; + +/** + * Heuristic for `: type =` annotations (mirrors the old `type_annotation` + * pattern): from the `:` at `i`, scans over a type expression with balanced + * `<>`/`[]`/`{}`/`()`, succeeding at a top-level `=` (not `==`/`=>`). + * Returns the exclusive end of the type text, or -1. + */ +const scan_type_annotation = (text: string, i: number, end: number): number => { + const limit = i + MAX_SCAN < end ? i + MAX_SCAN : end; + let angle = 0; + let square = 0; + let j = i + 1; + while (j < limit) { + const c = text.charCodeAt(j); + if (c === 61) { + // `=` + if (angle === 0 && square === 0) { + const c2 = text.charCodeAt(j + 1); + if (c2 === 61 || c2 === 62) return -1; + const c0 = text.charCodeAt(j - 1); + if (c0 === 33 || c0 === 60 || c0 === 62) return -1; // != <= >= + return j; + } + j++; + } else if (c === 60) { + angle++; + j++; + } else if (c === 62) { + if (angle === 0) return -1; + angle--; + j++; + } else if (c === 91) { + square++; + j++; + } else if (c === 93) { + if (square === 0) return -1; + square--; + j++; + } else if ( + angle === 0 && + square === 0 && + (c === 59 || c === 44 || c === 123 || c === 125 || c === 40 || c === 41) + ) { + // top-level statement/grouping chars end the candidate type text — + // object/function types are not annotation targets here (parity + // with the old pattern, which also rejected them; prevents the + // scan from running away across statement boundaries) + return -1; + } else if (c === 34 || c === 39) { + j = skip_quoted(text, j, end, c); + } else { + j++; + } + } + return -1; +}; + +/** + * Scans a multi-char operator at `i`, returning its length. + */ +const scan_operator = (text: string, i: number, end: number): number => { + const c = text.charCodeAt(i); + const c2 = i + 1 < end ? text.charCodeAt(i + 1) : 0; + const c3 = i + 2 < end ? text.charCodeAt(i + 2) : 0; + const c4 = i + 3 < end ? text.charCodeAt(i + 3) : 0; + switch (c) { + case 43: // + + return c2 === 43 || c2 === 61 ? 2 : 1; + case 45: // - + return c2 === 45 || c2 === 61 ? 2 : 1; + case 42: // * + if (c2 === 42) return c3 === 61 ? 3 : 2; + return c2 === 61 ? 2 : 1; + case 37: // % + return c2 === 61 ? 2 : 1; + case 38: // & + if (c2 === 38) return c3 === 61 ? 3 : 2; + return c2 === 61 ? 2 : 1; + case 124: // | + if (c2 === 124) return c3 === 61 ? 3 : 2; + return c2 === 61 ? 2 : 1; + case 94: // ^ + return c2 === 61 ? 2 : 1; + case 61: // = + if (c2 === 61) return c3 === 61 ? 3 : 2; + return c2 === 62 ? 2 : 1; // => + case 33: // ! + if (c2 === 61) return c3 === 61 ? 3 : 2; + return 1; + case 60: // < + if (c2 === 60) return c3 === 61 ? 3 : 2; + return c2 === 61 ? 2 : 1; + case 62: // > + if (c2 === 62) { + if (c3 === 62) return c4 === 61 ? 4 : 3; + return c3 === 61 ? 3 : 2; + } + return c2 === 61 ? 2 : 1; + case 63: // ? + if (c2 === 63) return c3 === 61 ? 3 : 2; + return c2 === 46 ? 2 : 1; // ?. + case 47: // / + return c2 === 61 ? 2 : 1; + default: + return 1; // ~ : + } +}; + +/** + * Lexes a template literal starting at the backtick at `i`, returning the new + * position. Interpolations recurse through the full lexer. + */ +const lex_ts_template = (l: Lexer, i: number, to: number, js: boolean): number => { + const {text} = l; + l.open(T_TEMPLATE_STRING, i); + l.leaf(T_TEMPLATE_PUNCTUATION, i, i + 1); + let j = i + 1; + let chunk_start = j; + let closed = false; + while (j < to) { + const c = text.charCodeAt(j); + if (c === 92) { + j += 2; + } else if (c === 96) { + l.leaf(T_STRING, chunk_start, j); + l.leaf(T_TEMPLATE_PUNCTUATION, j, j + 1); + j++; + closed = true; + break; + } else if (c === 36 && text.charCodeAt(j + 1) === 123) { + l.leaf(T_STRING, chunk_start, j); + const close = scan_balanced_braces(text, j + 1, to); + const inner_end = close === -1 ? to : close; + l.open(T_INTERPOLATION, j); + l.leaf(T_INTERPOLATION_PUNCTUATION, j, j + 2); + lex_ts_window(l, j + 2, inner_end, false, js); + if (close === -1) { + l.close(to); + j = to; + } else { + l.leaf(T_INTERPOLATION_PUNCTUATION, close, close + 1); + l.close(close + 1); + j = close + 1; + } + chunk_start = j; + } else { + j++; + } + } + if (!closed) { + if (j > to) j = to; + l.leaf(T_STRING, chunk_start, j); + } + l.close(j); + return j; +}; + +/** + * The core window lexer. `type_mode` switches capitalized identifiers to + * `type_name` (used for generics and type annotations). + */ +const lex_ts_window = ( + l: Lexer, + from: number, + to: number, + type_mode: boolean, + js: boolean, +): void => { + const {text} = l; + let i = from; + let prev = P_NONE; + let prev_code = 0; // last significant punctuation char, for string-property detection + let class_ctx = false; + let as_ctx = false; + let import_ctx = false; + + while (i < to) { + const c = text.charCodeAt(i); + if (is_space(c)) { + i++; + continue; + } + + // identifiers (including `#private`) + if (is_ident_start(c) || (c === 35 && is_ident_start(text.charCodeAt(i + 1)))) { + const start = i; + const ident_end = scan_ident(text, c === 35 ? i + 1 : i, to); + const was_class_ctx = class_ctx; + const was_as_ctx = as_ctx; + const was_import_ctx = import_ctx; + const was_dot = prev === P_DOT; + class_ctx = as_ctx = import_ctx = false; + prev = P_VALUE; + prev_code = 0; + i = ident_end; + + const word = text.slice(start, ident_end); + const kind = c === 35 || was_dot ? undefined : WORDS.get(word); + + if (was_class_ctx && kind === undefined && c !== 35) { + // class-name chain: `Foo`, `a.b.Foo`, optionally with generics + l.leaf(T_CLASS_NAME, start, ident_end); + while (text.charCodeAt(i) === 46 && is_ident_start(text.charCodeAt(i + 1)) && i + 1 < to) { + l.leaf(T_PUNCTUATION, i, i + 1); + const seg_end = scan_ident(text, i + 1, to); + l.leaf(T_CLASS_NAME, i + 1, seg_end); + i = seg_end; + } + if (!js) { + const angle_start = skip_space(text, i, to); + if (text.charCodeAt(angle_start) === 60) { + const angle_end = scan_balanced_angle(text, angle_start, to); + if (angle_end !== -1) { + lex_ts_window(l, angle_start, angle_end + 1, true, js); + i = angle_end + 1; + } + } + } + continue; + } + + if (was_as_ctx && kind === undefined && c !== 35 && !js && !BUILTIN_WORDS.has(word)) { + // `x as Foo` — but `as unknown`/`as string` keep their builtin type + l.leaf(T_TYPE_ASSERTION, start, ident_end); + continue; + } + + // keyword classification (with contextual lookaheads) + if (kind !== undefined) { + let keyword_id = 0; + switch (kind) { + case K_KEYWORD: + keyword_id = T_KEYWORD; + break; + case K_SPECIAL: + keyword_id = T_SPECIAL_KEYWORD; + break; + case K_TS: + if (!js) keyword_id = T_KEYWORD; + break; + case K_ASYNC: { + const n = text.charCodeAt(skip_space(text, ident_end, to)); + if (n === 40 || n === 42 || is_ident_start(n) || Number.isNaN(n)) { + keyword_id = T_KEYWORD; + } + break; + } + case K_GET_SET: { + const n = text.charCodeAt(skip_space(text, ident_end, to)); + if (n === 35 || n === 91 || is_ident_start(n) || Number.isNaN(n)) { + keyword_id = T_KEYWORD; + } + break; + } + case K_ASSERT: { + if (text.charCodeAt(skip_space(text, ident_end, to)) === 123) { + keyword_id = T_KEYWORD; + } + break; + } + case K_TYPE_WORD: { + if (js) break; + const n = text.charCodeAt(skip_space(text, ident_end, to)); + // `import type {…}` / `export type * from …` — a type-only + // import/export modifier, not a type-alias declaration + if (was_import_ctx && (n === 123 || n === 42)) { + l.leaf(T_IMPORT_TYPE_KEYWORD, start, ident_end); + continue; + } + if (n === 123 || n === 42 || is_ident_start(n)) { + keyword_id = T_KEYWORD; + } + break; + } + case K_TS_COND: { + if (js) break; + const n = text.charCodeAt(skip_space(text, ident_end, to)); + if (n === 123 || is_ident_start(n) || Number.isNaN(n)) { + keyword_id = T_KEYWORD; + } + break; + } + case K_BOOLEAN: + l.leaf(T_BOOLEAN, start, ident_end); + continue; + case K_NUMBER_WORD: + l.leaf(T_NUMBER, start, ident_end); + continue; + } + if (keyword_id !== 0) { + l.leaf(keyword_id, start, ident_end); + if (CLASS_CTX_WORDS.has(word) && (word !== 'type' || !js)) class_ctx = true; + if (AS_WORDS.has(word) && !js) as_ctx = true; + if (IMPORT_WORDS.has(word)) import_ctx = true; + if (!VALUE_WORDS.has(word)) prev = P_NONE; + continue; + } + } + + // function-valued variables: `f = () => …`, `f: async x => …` + if (!type_mode && c !== 35 && is_function_variable(text, ident_end, to)) { + l.leaf(T_FUNCTION_VARIABLE, start, ident_end); + continue; + } + + // SCREAMING_CASE constants, then capitalized identifiers — both + // before the call lookahead, matching the old grammar's priority + if (is_upper(c)) { + let all_caps = true; + for (let k = start + 1; k < ident_end; k++) { + const cc = text.charCodeAt(k); + // `x` only after a digit (hex-ish constants like `A0x`), per the old pattern + if ( + !is_upper(cc) && + cc !== 95 && + !is_digit(cc) && + !(cc === 120 && is_digit(text.charCodeAt(k - 1))) + ) { + all_caps = false; + break; + } + } + if (type_mode) { + l.leaf(T_TYPE_NAME, start, ident_end); + } else if (all_caps) { + l.leaf(T_CONSTANT, start, ident_end); + } else { + l.leaf(T_CAPITALIZED, start, ident_end); + } + continue; + } + + // generic call: `foo(…)` + if (!js) { + const angle_start = skip_space(text, ident_end, to); + if (text.charCodeAt(angle_start) === 60) { + const angle_end = scan_balanced_angle(text, angle_start, to); + if (angle_end !== -1 && text.charCodeAt(skip_space(text, angle_end + 1, to)) === 40) { + l.open(T_GENERIC_FUNCTION, start); + l.leaf(T_FUNCTION, start, ident_end); + l.open(T_GENERIC, angle_start); + lex_ts_window(l, angle_start, angle_end + 1, true, js); + l.close(angle_end + 1); + l.close(angle_end + 1); + i = angle_end + 1; + continue; + } + } + } + + // call: `foo(…)` (also `#private(…)`) + if (text.charCodeAt(skip_space(text, ident_end, to)) === 40) { + l.leaf(T_FUNCTION, start, ident_end); + continue; + } + + if (kind === undefined && !was_dot && c !== 35 && BUILTIN_WORDS.has(word)) { + l.leaf(T_BUILTIN, start, ident_end); + continue; + } + + // plain identifier — no token + continue; + } + + // `/` — comment, regex literal, or division + if (c === 47) { + const c2 = i + 1 < to ? text.charCodeAt(i + 1) : 0; + if (c2 === 47) { + const line_end = scan_to_line_end(text, i, to); + l.leaf(T_COMMENT, i, line_end); + i = line_end; + continue; // comments are transparent — contexts survive + } + if (c2 === 42) { + const close = text.indexOf('*/', i + 2); + const comment_end = close === -1 || close + 2 > to ? to : close + 2; + l.leaf(T_COMMENT, i, comment_end); + i = comment_end; + continue; + } + class_ctx = as_ctx = import_ctx = false; + if (prev !== P_VALUE && prev !== P_DOT) { + // regex literal position + const body_end = scan_regex_body(text, i, to); + if (body_end !== -1) { + let flags_end = body_end + 1; + while (flags_end < to) { + const f = text.charCodeAt(flags_end); + if (f >= 97 && f <= 122) flags_end++; + else break; + } + l.open(T_REGEX, i); + l.leaf(T_REGEX_DELIMITER, i, i + 1); + l.leaf(T_REGEX_SOURCE, i + 1, body_end); + l.leaf(T_REGEX_DELIMITER, body_end, body_end + 1); + if (flags_end > body_end + 1) l.leaf(T_REGEX_FLAGS, body_end + 1, flags_end); + l.close(flags_end); + i = flags_end; + prev = P_VALUE; + prev_code = 0; + continue; + } + } + const op_len = scan_operator(text, i, to); + l.leaf(T_OPERATOR, i, i + op_len); + i += op_len; + prev = P_NONE; + prev_code = 0; + continue; + } + + // strings + if (c === 34 || c === 39) { + const str_end = scan_ts_string(text, i, to, c); + // `{"key": …}` / `, "key": …` — a string property key + let is_property = false; + if (!type_mode && (prev_code === 123 || prev_code === 44)) { + const after = skip_space(text, str_end, to); + is_property = text.charCodeAt(after) === 58; + } + l.leaf(is_property ? T_STRING_PROPERTY : T_STRING, i, str_end); + i = str_end; + prev = P_VALUE; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // template literals + if (c === 96) { + i = lex_ts_template(l, i, to, js); + prev = P_VALUE; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // numbers (also `.5` — the scan handles a leading `.` before digits) + if (is_digit(c) || (c === 46 && is_digit(text.charCodeAt(i + 1)))) { + const num_end = scan_ts_number(text, i, to); + l.leaf(T_NUMBER, i, num_end); + i = num_end; + prev = P_VALUE; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // `.` — member access or spread + if (c === 46) { + class_ctx = as_ctx = import_ctx = false; + if (text.charCodeAt(i + 1) === 46 && text.charCodeAt(i + 2) === 46) { + l.leaf(T_OPERATOR, i, i + 3); + i += 3; + prev = P_NONE; + prev_code = 0; + } else { + l.leaf(T_PUNCTUATION, i, i + 1); + i++; + prev = P_DOT; + prev_code = 46; + } + continue; + } + + // `@decorator` + if (c === 64 && is_ident_start(text.charCodeAt(i + 1))) { + const name_end = scan_ident(text, i + 1, to); + l.open(T_DECORATOR, i); + l.leaf(T_AT, i, i + 1); + l.leaf(T_DECORATOR_NAME, i + 1, name_end); + l.close(name_end); + i = name_end; + prev = P_NONE; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // `:` — type annotation (with initializer) or plain operator + if (c === 58) { + class_ctx = as_ctx = import_ctx = false; + if (!type_mode && !js) { + const type_end = scan_type_annotation(text, i, to); + if (type_end !== -1) { + const type_start = skip_space(text, i + 1, to); + let content_end = type_end; + while (content_end > type_start && is_space(text.charCodeAt(content_end - 1))) { + content_end--; + } + if (content_end > type_start) { + l.open(T_TYPE_ANNOTATION, i); + l.leaf(T_COLON, i, i + 1); + l.open(T_TYPE, type_start); + lex_ts_window(l, type_start, content_end, true, js); + l.close(content_end); + l.close(content_end); + i = type_end; + prev = P_NONE; + prev_code = 0; + continue; + } + } + } + l.leaf(T_OPERATOR, i, i + 1); + i++; + prev = P_NONE; + prev_code = 0; + continue; + } + + // punctuation + if ( + c === 123 || + c === 125 || + c === 91 || + c === 93 || + c === 40 || + c === 41 || + c === 59 || + c === 44 + ) { + l.leaf(T_PUNCTUATION, i, i + 1); + prev = c === 41 || c === 93 ? P_VALUE : P_NONE; + prev_code = c; + i++; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // operators + if ( + c === 43 || + c === 45 || + c === 42 || + c === 37 || + c === 38 || + c === 124 || + c === 94 || + c === 61 || + c === 33 || + c === 60 || + c === 62 || + c === 63 || + c === 126 + ) { + const op_len = scan_operator(text, i, to); + l.leaf(T_OPERATOR, i, i + op_len); + // `?.` is member access — the next word is a property, not a keyword + prev = c === 63 && op_len === 2 && text.charCodeAt(i + 1) === 46 ? P_DOT : P_NONE; + i += op_len; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + continue; + } + + // anything else is plain text + i++; + prev = P_NONE; + prev_code = 0; + class_ctx = as_ctx = import_ctx = false; + } +}; + +/** + * Scans a regex literal body from the `/` at `i`, returning the index of the + * closing `/`, or -1 when this isn't a regex literal. Handles character + * classes and escapes; regex literals never span lines. + */ +const scan_regex_body = (text: string, i: number, to: number): number => { + let j = i + 1; + let in_class = false; + let any = false; + while (j < to) { + const c = text.charCodeAt(j); + if (c === 92) { + j += 2; + any = true; + } else if (c === 10 || c === 13) { + return -1; + } else if (in_class) { + if (c === 93) in_class = false; + j++; + } else if (c === 91) { + in_class = true; + any = true; + j++; + } else if (c === 47) { + return any ? j : -1; // empty body means `//` (handled as comment anyway) + } else { + any = true; + j++; + } + } + return -1; +}; + +const create_lex_ts = + (js: boolean) => + (l: Lexer): void => { + let i = l.pos; + // hashbang at the very start of the document + if (i === 0 && l.text.charCodeAt(0) === 35 && l.text.charCodeAt(1) === 33) { + const line_end = scan_to_line_end(l.text, 0, l.end); + l.leaf(T_HASHBANG, 0, line_end); + i = line_end; + } + lex_ts_window(l, i, l.end, false, js); + l.pos = l.end; + }; + +/** + * The TypeScript language registration for the lexer engine. + */ +export const lexer_ts: SyntaxLang = {id: 'ts', aliases: ['typescript'], lex: create_lex_ts(false)}; diff --git a/src/lib/syntax_styler.ts b/src/lib/syntax_styler.ts index 8d98ac8d..9975e4bb 100644 --- a/src/lib/syntax_styler.ts +++ b/src/lib/syntax_styler.ts @@ -1,5 +1,6 @@ import {SyntaxToken, type SyntaxTokenStream} from './syntax_token.ts'; import {tokenize_syntax} from './tokenize_syntax.ts'; +import {lex_syntax, render_syntax_html, type LexedSyntax, type SyntaxLang} from './lexer.ts'; export type AddSyntaxGrammar = (syntax_styler: SyntaxStyler) => void; @@ -39,6 +40,44 @@ export class SyntaxStyler { // } // } + /** + * Languages ported to the hand-written lexer engine (`lexer.ts`). + * When a language is registered here, `stylize` routes through the lexer + * engine instead of the regex-grammar engine; the grammar registration in + * `langs` remains only for the unported paths (`tokenize`, embedded + * regions in unported grammars) until the old engine is deleted. + */ + lexer_langs: Map = new Map(); + + /** + * Registers a lexer-engine language (and its aliases). + */ + add_lexer_lang(lang: SyntaxLang): void { + this.lexer_langs.set(lang.id, lang); + if (lang.aliases) { + for (const alias of lang.aliases) { + this.lexer_langs.set(alias, lang); + } + } + } + + has_lexer_lang(id: string): boolean { + return this.lexer_langs.has(id); + } + + /** + * Lexes `text` with the lexer-engine language registered as `lang`, + * returning the flat token event stream. Throws when `lang` has no + * registered lexer — see `has_lexer_lang`. + */ + lex(text: string, lang: string): LexedSyntax { + const lexer_lang = this.lexer_langs.get(lang); + if (lexer_lang === undefined) { + throw Error(`The language "${lang}" has no lexer.`); + } + return lex_syntax(text, lexer_lang, this.lexer_langs); + } + add_lang(id: string, grammar: SyntaxGrammarRaw, aliases?: Array): void { // Normalize grammar once at registration for optimal runtime performance // Use a visited set to handle circular references @@ -129,14 +168,19 @@ export class SyntaxStyler { * stylize('var foo = 42;', 'ts-extended', extended); * ``` */ - stylize( - text: string, - lang: string, - grammar: SyntaxGrammar | undefined = this.get_lang(lang), - ): string { + stylize(text: string, lang: string, grammar?: SyntaxGrammar): string { + let resolved_grammar = grammar; + if (resolved_grammar === undefined) { + // lexer-engine languages take priority over their grammar registrations + const lexer_lang = this.lexer_langs.get(lang); + if (lexer_lang !== undefined) { + return render_syntax_html(lex_syntax(text, lexer_lang, this.lexer_langs)); + } + resolved_grammar = this.get_lang(lang); + } // stringify with the post-hook `lang`, which a `before_tokenize` hook may // have rewritten (it flows into each token's `wrap` hook context) - const c = this.#tokenize_hooked(text, lang, grammar); + const c = this.#tokenize_hooked(text, lang, resolved_grammar); return this.stringify_token(c.tokens, c.lang); } diff --git a/src/lib/syntax_styler_global.ts b/src/lib/syntax_styler_global.ts index 0159349f..a4bdb6c1 100644 --- a/src/lib/syntax_styler_global.ts +++ b/src/lib/syntax_styler_global.ts @@ -1,4 +1,8 @@ import {SyntaxStyler} from './syntax_styler.ts'; +import {lexer_json} from './lexer_json.ts'; +import {lexer_ts} from './lexer_ts.ts'; +import {lexer_css} from './lexer_css.ts'; +import {lexer_bash} from './lexer_bash.ts'; import {add_grammar_markup} from './grammar_markup.ts'; import {add_grammar_css} from './grammar_css.ts'; import {add_grammar_clike} from './grammar_clike.ts'; @@ -30,3 +34,11 @@ add_grammar_svelte(syntax_styler_global); add_grammar_json(syntax_styler_global); add_grammar_bash(syntax_styler_global); // before markdown — markdown references bash for fenced code blocks add_grammar_markdown(syntax_styler_global); + +// Languages ported to the lexer engine — these take priority over the grammar +// registrations above in `stylize`; the grammars remain for the unported +// paths (`tokenize`, embedded regions in unported grammars like md fences). +syntax_styler_global.add_lexer_lang(lexer_json); +syntax_styler_global.add_lexer_lang(lexer_ts); +syntax_styler_global.add_lexer_lang(lexer_css); +syntax_styler_global.add_lexer_lang(lexer_bash); diff --git a/src/test/fixtures/generated/bash/bash_complex.html b/src/test/fixtures/generated/bash/bash_complex.html index 3083090c..21d6e88d 100644 --- a/src/test/fixtures/generated/bash/bash_complex.html +++ b/src/test/fixtures/generated/bash/bash_complex.html @@ -28,16 +28,16 @@ echo ${!map[@]} # Control flow -if [ -f "$name" ]; then +if [ -f "$name" ]; then echo "file exists" -elif [[ $name == "world" ]]; then +elif [[ $name == "world" ]]; then echo "hello world" else echo "something else" fi # Regex match in [[ -if [[ $name =~ ^[a-z]+$ ]]; then +if [[ $name =~ ^[a-z]+$ ]]; then echo "lowercase" fi @@ -46,7 +46,7 @@ echo $i done -for ((i = 0; i < 10; i++)); do +for ((i = 0; i < 10; i++)); do echo $i done @@ -80,7 +80,7 @@ esac # Functions -greet() { +greet() { local greeting="Hello" echo "$greeting, $1!" return 0 @@ -95,9 +95,9 @@ files=$(ls -la | grep "\.txt$") # Arithmetic expansion -result=$(( 2 + 3 * 4 )) -(( count++ )) -(( x = y > 0 ? y : -y )) +result=$(( 2 + 3 * 4 )) +(( count++ )) +(( x = y > 0 ? y : -y )) # Here-document cat <<EOF diff --git a/src/test/fixtures/generated/bash/bash_complex.txt b/src/test/fixtures/generated/bash/bash_complex.txt index e379230d..ff746faf 100644 --- a/src/test/fixtures/generated/bash/bash_complex.txt +++ b/src/test/fixtures/generated/bash/bash_complex.txt @@ -1,9 +1,9 @@ === STATS === Sample length: 2726 characters -Total tokens: 354 +Total tokens: 338 Token types (15 unique): - punctuation: 77 + punctuation: 62 builtin: 57 keyword: 45 string: 40 @@ -11,8 +11,8 @@ Token types (15 unique): variable: 30 comment: 26 number: 23 - command_substitution: 6 heredoc_delimiter: 6 + command_substitution: 5 boolean: 3 heredoc: 3 file_descriptor: 3 @@ -69,20 +69,16 @@ Token types (15 unique): 436-437 punctuation [ 441-448 string "$name" 442-447 variable $name - 449-450 punctuation ] - 450-451 punctuation ; + 449-451 punctuation ]; 452-456 keyword then 458-462 builtin echo 463-476 string "file exists" 477-481 keyword elif - 482-483 punctuation [ - 483-484 punctuation [ + 482-484 punctuation [[ 485-490 variable $name 491-493 operator == 494-501 string "world" - 502-503 punctuation ] - 503-504 punctuation ] - 504-505 punctuation ; + 502-505 punctuation ]]; 506-510 keyword then 512-516 builtin echo 517-530 string "hello world" @@ -92,15 +88,12 @@ Token types (15 unique): 559-561 keyword fi 563-582 comment # Regex match in [[ 583-585 keyword if - 586-587 punctuation [ - 587-588 punctuation [ + 586-588 punctuation [[ 589-594 variable $name 595-597 operator =~ 599-600 punctuation [ 603-604 punctuation ] - 607-608 punctuation ] - 608-609 punctuation ] - 609-610 punctuation ; + 607-610 punctuation ]]; 611-615 keyword then 617-621 builtin echo 622-633 string "lowercase" @@ -117,16 +110,13 @@ Token types (15 unique): 671-673 variable $i 674-678 keyword done 680-683 keyword for - 684-685 punctuation ( - 685-686 punctuation ( + 684-686 punctuation (( 690-691 number 0 691-692 punctuation ; 695-696 operator < 697-699 number 10 699-700 punctuation ; - 704-705 punctuation ) - 705-706 punctuation ) - 706-707 punctuation ; + 704-707 punctuation )); 708-710 keyword do 712-716 builtin echo 717-719 variable $i @@ -188,8 +178,7 @@ Token types (15 unique): 1062-1066 keyword esac 1068-1079 comment # Functions 1080-1085 function greet -1085-1086 punctuation ( -1086-1087 punctuation ) +1085-1087 punctuation () 1088-1089 punctuation { 1091-1096 keyword local 1106-1113 string "Hello" @@ -216,22 +205,17 @@ Token types (15 unique): 1261-1269 string "\.txt$" 1269-1270 punctuation ) 1272-1294 comment # Arithmetic expansion -1302-1318 command_substitution $(( 2 + 3 * 4 )) -1302-1304 punctuation $( +1302-1305 punctuation $(( 1306-1307 number 2 1310-1311 number 3 1314-1315 number 4 -1317-1318 punctuation ) -1319-1320 punctuation ( -1320-1321 punctuation ( -1330-1331 punctuation ) -1331-1332 punctuation ) -1333-1334 punctuation ( -1334-1335 punctuation ( +1316-1318 punctuation )) +1319-1321 punctuation (( +1330-1332 punctuation )) +1333-1335 punctuation (( 1342-1343 operator > 1344-1345 number 0 -1355-1356 punctuation ) -1356-1357 punctuation ) +1355-1357 punctuation )) 1359-1374 comment # Here-document 1379-1440 heredoc <} p { - box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); + box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); } /* comment */ @@ -31,7 +31,7 @@ @media (max-width: 600px) { body { - background-color: light-dark(lightblue, darkblue); + background-color: light-dark(lightblue, darkblue); } } diff --git a/src/test/fixtures/generated/css/css_complex.txt b/src/test/fixtures/generated/css/css_complex.txt index 37a7a612..6bf44c21 100644 --- a/src/test/fixtures/generated/css/css_complex.txt +++ b/src/test/fixtures/generated/css/css_complex.txt @@ -1,9 +1,9 @@ === STATS === Sample length: 502 characters -Total tokens: 75 +Total tokens: 73 Token types (9 unique): - punctuation: 47 + punctuation: 45 property: 9 selector: 8 function: 3 @@ -35,8 +35,7 @@ Token types (9 unique): 98-99 punctuation , 101-102 punctuation , 104-105 punctuation , - 109-110 punctuation ) - 110-111 punctuation ; + 109-111 punctuation ); 112-113 punctuation } 115-128 comment /* comment */ 130-176 comment /*\nmulti\n.line {\n\ti: 100px\n\n\n\n@media*/ @@ -66,8 +65,7 @@ Token types (9 unique): 296-306 function light-dark 306-307 punctuation ( 316-317 punctuation , - 326-327 punctuation ) - 327-328 punctuation ; + 326-328 punctuation ); 330-331 punctuation } 332-333 punctuation } 335-351 selector .content::before diff --git a/src/test/fixtures/generated/json/json_complex.html b/src/test/fixtures/generated/json/json_complex.html index e69a58f0..d5d0aa5d 100644 --- a/src/test/fixtures/generated/json/json_complex.html +++ b/src/test/fixtures/generated/json/json_complex.html @@ -6,9 +6,9 @@ "empty": "", "escaped": "quote: \"test\" and backslash: \\", "object": { - "array": [1, "b", false], - "strings": ["1", "2", "3"], - "mixed": ["start", 123, true, "middle", null, "end"], - "nested": [["a", "str", ""], {"key": "nested value"}] + "array": [1, "b", false], + "strings": ["1", "2", "3"], + "mixed": ["start", 123, true, "middle", null, "end"], + "nested": [["a", "str", ""], {"key": "nested value"}] } } diff --git a/src/test/fixtures/generated/json/json_complex.txt b/src/test/fixtures/generated/json/json_complex.txt index e32ee2b3..65b2c771 100644 --- a/src/test/fixtures/generated/json/json_complex.txt +++ b/src/test/fixtures/generated/json/json_complex.txt @@ -1,9 +1,9 @@ === STATS === Sample length: 327 characters -Total tokens: 83 +Total tokens: 77 Token types (7 unique): - punctuation: 37 + punctuation: 31 string: 14 property: 12 operator: 12 @@ -48,8 +48,7 @@ Token types (7 unique): 167-170 string "b" 170-171 punctuation , 172-177 boolean false - 177-178 punctuation ] - 178-179 punctuation , + 177-179 punctuation ], 182-191 property "strings" 191-192 operator : 193-194 punctuation [ @@ -58,8 +57,7 @@ Token types (7 unique): 199-202 string "2" 202-203 punctuation , 204-207 string "3" - 207-208 punctuation ] - 208-209 punctuation , + 207-209 punctuation ], 212-219 property "mixed" 219-220 operator : 221-222 punctuation [ @@ -74,24 +72,20 @@ Token types (7 unique): 252-256 null null 256-257 punctuation , 258-263 string "end" - 263-264 punctuation ] - 264-265 punctuation , + 263-265 punctuation ], 268-276 property "nested" 276-277 operator : - 278-279 punctuation [ - 279-280 punctuation [ + 278-280 punctuation [[ 280-283 string "a" 283-284 punctuation , 285-290 string "str" 290-291 punctuation , 292-294 string "" - 294-295 punctuation ] - 295-296 punctuation , + 294-296 punctuation ], 297-298 punctuation { 298-303 property "key" 303-304 operator : 305-319 string "nested value" - 319-320 punctuation } - 320-321 punctuation ] + 319-321 punctuation }] 323-324 punctuation } 325-326 punctuation } diff --git a/src/test/fixtures/generated/ts/ts_complex.html b/src/test/fixtures/generated/ts/ts_complex.html index 98c999d9..db2906d3 100644 --- a/src/test/fixtures/generated/ts/ts_complex.html +++ b/src/test/fixtures/generated/ts/ts_complex.html @@ -1,105 +1,105 @@ const a = 1; -const b: string = 'b'; +const b: string = 'b'; const c = true; -export type SomeType = 1 | 'b' | true; +export type SomeType = 1 | 'b' | true; declare const some_decorator: any; -abstract class Base { - abstract abstract_method(): void; +abstract class Base { + abstract abstract_method(): void; } /* eslint-disable no-console */ @some_decorator -class D extends Base { - readonly d1: string = 'd'; +class D extends Base { + readonly d1: string = 'd'; d2: number; - d3 = $state(null); + d3 = $state(null); @some_decorator decorated = true; constructor(d2: number) { - super(); + super(); this.d2 = d2; } - abstract_method(): void { + abstract_method(): void { // implementation } - @some_decorator('example', {option: true}) - class_method(): string { + @some_decorator('example', {option: true}) + class_method(): string { return `Hello, ${this.d1}`; } - instance_method = (): void => { + instance_method = (): void => { /* ... */ let i = 0; do { i++; - } while (i < 3); + } while (i < 3); for (const c2 of this.d1) { if (c2 === 'd') continue; if (!c2) break; - this.#private_method(a, c2); + this.#private_method(a, c2); } switch (this.d1) { case 'a': - console.log('case a'); + console.log('case a'); break; case 'b': case 'c': - console.log('case b or c'); + console.log('case b or c'); break; default: - console.log('default'); + console.log('default'); } - const obj: {has_d1?: boolean; is_d: boolean} = { + const obj: {has_d1?: boolean; is_d: boolean} = { has_d1: 'd1' in this, - is_d: this instanceof D, - }; + is_d: this instanceof D, + }; delete obj.has_d1; // foo - }; + }; #private_method(a2: number, c2: any): void { - throw new Error(`${this.d1} + throw new Error(`${this.d1} multiline etc ${a2 + c2} - `); + `); } - *generator(): Generator<number | Array<number>> { + *generator(): Generator<number | Array<number>> { yield 1; - yield* [2, 3]; + yield* [2, 3]; } - async *async_generator(): AsyncGenerator<number> { - yield await Promise.resolve(4); + async *async_generator(): AsyncGenerator<number> { + yield await Promise.resolve(4); } - protected async protected_method(): Promise<void> { + protected async protected_method(): Promise<void> { try { - await new Promise((resolve) => setTimeout(resolve, 100)); - if (Math.random() > 0.5) { - console.log(new Date()); - } else if (Math.random() > 0.2) { - console.log('else if branch'); + await new Promise((resolve) => setTimeout(resolve, 100)); + if (Math.random() > 0.5) { + console.log(new Date()); + } else if (Math.random() > 0.2) { + console.log('else if branch'); } else { - console.log('else branch'); + console.log('else branch'); } } catch (error: unknown) { - console.error(error); + console.error(error); } finally { - console.log('finally block'); + console.log('finally block'); } } } @@ -116,28 +116,28 @@ * JSDoc comment */ -import {sample_langs, type SampleLang} from '$lib/code_sample.ts'; -import * as A from '$lib/code_sample.ts'; +import {sample_langs, type SampleLang} from '$lib/code_sample.ts'; +import * as A from '$lib/code_sample.ts'; -export {a, A, b, c, D}; +export {a, A, b, c, D}; -sample_langs as unknown as SampleLang satisfies SampleLang; +sample_langs as unknown as SampleLang satisfies SampleLang; -export interface SomeE<T = null> { +export interface SomeE<T = null> { name: string; age: number; - t?: T; + t?: T; } -const e: {name: string; age: number} = {name: 'A. H.', age: 100}; -const v = [['', e]] as const; -export const some_e: Map<string, SomeE> = new Map(v); +const e: {name: string; age: number} = {name: 'A. H.', age: 100}; +const v = [['', e]] as const; +export const some_e: Map<string, SomeE> = new Map(v); export function add(x: number, y: number): number { return x + y; } -export const plus = (a: any, b: any): any => a + b; +export const plus = (a: any, b: any): any => a + b; // boundary test cases export const str_with_keywords = 'const class function string'; diff --git a/src/test/fixtures/generated/ts/ts_complex.txt b/src/test/fixtures/generated/ts/ts_complex.txt index 48e7a122..c0065ade 100644 --- a/src/test/fixtures/generated/ts/ts_complex.txt +++ b/src/test/fixtures/generated/ts/ts_complex.txt @@ -1,31 +1,33 @@ === STATS === Sample length: 2883 characters -Total tokens: 632 +Total tokens: 577 -Token types (27 unique): - punctuation: 233 +Token types (29 unique): + punctuation: 195 operator: 90 keyword: 56 special_keyword: 48 builtin: 34 function: 26 string: 25 - capitalized_identifier: 20 number: 14 class_name: 11 comment: 11 interpolation_punctuation: 8 - constant: 7 + capitalized_identifier: 7 template_punctuation: 6 - type_annotation: 5 - :: 5 - type: 5 boolean: 4 interpolation: 4 regex_delimiter: 4 + type_annotation: 3 + :: 3 + type: 3 decorator: 3 at: 3 template_string: 3 + type_assertion: 3 + constant: 3 + type_name: 3 function_variable: 2 regex: 2 regex_source: 2 @@ -37,9 +39,9 @@ Token types (27 unique): 10-11 number 1 11-12 punctuation ; 14-19 keyword const - 21-30 type_annotation : string + 21-29 type_annotation : string 21-22 : : - 22-30 type string + 23-29 type string 23-29 builtin string 30-31 operator = 32-35 string 'b' @@ -51,7 +53,6 @@ Token types (27 unique): 55-61 special_keyword export 62-66 keyword type 67-75 class_name SomeType - 67-75 capitalized_identifier SomeType 76-77 operator = 78-79 number 1 80-81 operator | @@ -67,12 +68,10 @@ Token types (27 unique): 131-139 keyword abstract 140-145 keyword class 146-150 class_name Base - 146-150 capitalized_identifier Base 151-152 punctuation { 154-162 keyword abstract 163-178 function abstract_method - 178-179 punctuation ( - 179-180 punctuation ) + 178-180 punctuation () 180-181 operator : 182-186 keyword void 186-187 punctuation ; @@ -83,15 +82,13 @@ Token types (27 unique): 225-239 function some_decorator 240-245 keyword class 246-247 class_name D - 246-247 constant D 248-255 keyword extends 256-260 class_name Base - 256-260 capitalized_identifier Base 261-262 punctuation { 264-272 keyword readonly - 275-284 type_annotation : string + 275-283 type_annotation : string 275-276 : : - 276-284 type string + 277-283 type string 277-283 builtin string 284-285 operator = 286-289 string 'd' @@ -103,8 +100,7 @@ Token types (27 unique): 310-316 function $state 316-317 punctuation ( 317-321 keyword null - 321-322 punctuation ) - 322-323 punctuation ; + 321-323 punctuation ); 326-341 decorator @some_decorator 326-327 at @ 327-341 function some_decorator @@ -118,17 +114,14 @@ Token types (27 unique): 385-386 punctuation ) 387-388 punctuation { 391-396 keyword super - 396-397 punctuation ( - 397-398 punctuation ) - 398-399 punctuation ; + 396-399 punctuation (); 402-406 keyword this 406-407 punctuation . 410-411 operator = 414-415 punctuation ; 417-418 punctuation } 421-436 function abstract_method - 436-437 punctuation ( - 437-438 punctuation ) + 436-438 punctuation () 438-439 operator : 440-444 keyword void 445-446 punctuation { @@ -143,11 +136,9 @@ Token types (27 unique): 499-500 punctuation { 506-507 operator : 508-512 boolean true - 512-513 punctuation } - 513-514 punctuation ) + 512-514 punctuation }) 516-528 function class_method - 528-529 punctuation ( - 529-530 punctuation ) + 528-530 punctuation () 530-531 operator : 532-538 builtin string 539-540 punctuation { @@ -165,11 +156,8 @@ Token types (27 unique): 572-573 punctuation } 576-591 function_variable instance_method 592-593 operator = - 594-595 punctuation ( - 595-596 punctuation ) - 596-603 type_annotation : void - 596-597 : : - 597-603 type void + 594-596 punctuation () + 596-597 operator : 598-602 keyword void 603-605 operator => 606-607 punctuation { @@ -187,8 +175,7 @@ Token types (27 unique): 658-659 punctuation ( 661-662 operator < 663-664 number 3 - 664-665 punctuation ) - 665-666 punctuation ; + 664-666 punctuation ); 670-673 special_keyword for 674-675 punctuation ( 675-680 keyword const @@ -215,8 +202,7 @@ Token types (27 unique): 754-769 function #private_method 769-770 punctuation ( 771-772 punctuation , - 775-776 punctuation ) - 776-777 punctuation ; + 775-777 punctuation ); 780-781 punctuation } 785-791 special_keyword switch 792-793 punctuation ( @@ -232,8 +218,7 @@ Token types (27 unique): 829-832 function log 832-833 punctuation ( 833-841 string 'case a' - 841-842 punctuation ) - 842-843 punctuation ; + 841-843 punctuation ); 848-853 special_keyword break 853-854 punctuation ; 858-862 special_keyword case @@ -247,8 +232,7 @@ Token types (27 unique): 893-896 function log 896-897 punctuation ( 897-910 string 'case b or c' - 910-911 punctuation ) - 911-912 punctuation ; + 910-912 punctuation ); 917-922 special_keyword break 922-923 punctuation ; 927-934 special_keyword default @@ -258,14 +242,12 @@ Token types (27 unique): 948-951 function log 951-952 punctuation ( 952-961 string 'default' - 961-962 punctuation ) - 962-963 punctuation ; + 961-963 punctuation ); 966-967 punctuation } 971-976 keyword const 980-981 operator : 982-983 punctuation { - 989-990 operator ? - 990-991 operator : + 989-991 operator ?: 992-999 builtin boolean 999-1000 punctuation ; 1005-1006 operator : @@ -282,16 +264,13 @@ Token types (27 unique): 1054-1058 keyword this 1059-1069 keyword instanceof 1070-1071 class_name D -1070-1071 constant D 1071-1072 punctuation , -1075-1076 punctuation } -1076-1077 punctuation ; +1075-1077 punctuation }; 1080-1086 keyword delete 1090-1091 punctuation . 1097-1098 punctuation ; 1101-1107 comment // foo -1109-1110 punctuation } -1110-1111 punctuation ; +1109-1111 punctuation }; 1114-1129 function #private_method 1129-1130 punctuation ( 1132-1133 operator : @@ -306,7 +285,6 @@ Token types (27 unique): 1161-1166 special_keyword throw 1167-1170 keyword new 1171-1176 class_name Error -1171-1176 capitalized_identifier Error 1176-1177 punctuation ( 1177-1223 template_string `${this.d1}\n\t\t\tmultiline\n\t\t\tetc ${a2 + c2}\n\t\t` 1177-1178 template_punctuation ` @@ -322,13 +300,11 @@ Token types (27 unique): 1218-1219 interpolation_punctuation } 1219-1222 string \n\t\t 1222-1223 template_punctuation ` -1223-1224 punctuation ) -1224-1225 punctuation ; +1223-1225 punctuation ); 1227-1228 punctuation } 1231-1232 operator * 1232-1241 function generator -1241-1242 punctuation ( -1242-1243 punctuation ) +1241-1243 punctuation () 1243-1244 operator : 1245-1254 capitalized_identifier Generator 1254-1255 operator < @@ -348,14 +324,12 @@ Token types (27 unique): 1302-1303 number 2 1303-1304 punctuation , 1305-1306 number 3 -1306-1307 punctuation ] -1307-1308 punctuation ; +1306-1308 punctuation ]; 1310-1311 punctuation } 1314-1319 keyword async 1320-1321 operator * 1321-1336 function async_generator -1336-1337 punctuation ( -1337-1338 punctuation ) +1336-1338 punctuation () 1338-1339 operator : 1340-1354 capitalized_identifier AsyncGenerator 1354-1355 operator < @@ -369,14 +343,12 @@ Token types (27 unique): 1387-1394 function resolve 1394-1395 punctuation ( 1395-1396 number 4 -1396-1397 punctuation ) -1397-1398 punctuation ; +1396-1398 punctuation ); 1400-1401 punctuation } 1404-1413 keyword protected 1414-1419 keyword async 1420-1436 function protected_method -1436-1437 punctuation ( -1437-1438 punctuation ) +1436-1438 punctuation () 1438-1439 operator : 1440-1447 capitalized_identifier Promise 1447-1448 operator < @@ -388,25 +360,20 @@ Token types (27 unique): 1467-1472 special_keyword await 1473-1476 keyword new 1477-1484 class_name Promise -1477-1484 capitalized_identifier Promise -1484-1485 punctuation ( -1485-1486 punctuation ( +1484-1486 punctuation (( 1493-1494 punctuation ) 1495-1497 operator => 1498-1508 function setTimeout 1508-1509 punctuation ( 1516-1517 punctuation , 1518-1521 number 100 -1521-1522 punctuation ) -1522-1523 punctuation ) -1523-1524 punctuation ; +1521-1524 punctuation )); 1528-1530 special_keyword if 1531-1532 punctuation ( 1532-1536 capitalized_identifier Math 1536-1537 punctuation . 1537-1543 function random -1543-1544 punctuation ( -1544-1545 punctuation ) +1543-1545 punctuation () 1546-1547 operator > 1548-1551 number 0.5 1551-1552 punctuation ) @@ -417,11 +384,7 @@ Token types (27 unique): 1570-1571 punctuation ( 1571-1574 keyword new 1575-1579 class_name Date -1575-1579 capitalized_identifier Date -1579-1580 punctuation ( -1580-1581 punctuation ) -1581-1582 punctuation ) -1582-1583 punctuation ; +1579-1583 punctuation ()); 1587-1588 punctuation } 1589-1593 special_keyword else 1594-1596 special_keyword if @@ -429,8 +392,7 @@ Token types (27 unique): 1598-1602 capitalized_identifier Math 1602-1603 punctuation . 1603-1609 function random -1609-1610 punctuation ( -1610-1611 punctuation ) +1609-1611 punctuation () 1612-1613 operator > 1614-1617 number 0.2 1617-1618 punctuation ) @@ -440,8 +402,7 @@ Token types (27 unique): 1633-1636 function log 1636-1637 punctuation ( 1637-1653 string 'else if branch' -1653-1654 punctuation ) -1654-1655 punctuation ; +1653-1655 punctuation ); 1659-1660 punctuation } 1661-1665 special_keyword else 1666-1667 punctuation { @@ -450,8 +411,7 @@ Token types (27 unique): 1680-1683 function log 1683-1684 punctuation ( 1684-1697 string 'else branch' -1697-1698 punctuation ) -1698-1699 punctuation ; +1697-1699 punctuation ); 1703-1704 punctuation } 1707-1708 punctuation } 1709-1714 special_keyword catch @@ -464,8 +424,7 @@ Token types (27 unique): 1744-1745 punctuation . 1745-1750 function error 1750-1751 punctuation ( -1756-1757 punctuation ) -1757-1758 punctuation ; +1756-1758 punctuation ); 1761-1762 punctuation } 1763-1770 special_keyword finally 1771-1772 punctuation { @@ -474,8 +433,7 @@ Token types (27 unique): 1784-1787 function log 1787-1788 punctuation ( 1788-1803 string 'finally block' -1803-1804 punctuation ) -1804-1805 punctuation ; +1803-1805 punctuation ); 1808-1809 punctuation } 1811-1812 punctuation } 1813-1814 punctuation } @@ -487,7 +445,6 @@ Token types (27 unique): 1919-1920 punctuation , 1921-1925 keyword type 1926-1936 class_name SampleLang -1926-1936 capitalized_identifier SampleLang 1936-1937 punctuation } 1938-1942 special_keyword from 1943-1964 string '$lib/code_sample.ts' @@ -495,7 +452,7 @@ Token types (27 unique): 1966-1972 special_keyword import 1973-1974 operator * 1975-1977 special_keyword as -1978-1979 constant A +1978-1979 type_assertion A 1980-1984 special_keyword from 1985-2006 string '$lib/code_sample.ts' 2006-2007 punctuation ; @@ -507,21 +464,19 @@ Token types (27 unique): 2024-2025 punctuation , 2027-2028 punctuation , 2029-2030 constant D -2030-2031 punctuation } -2031-2032 punctuation ; +2030-2032 punctuation }; 2047-2049 special_keyword as 2050-2057 builtin unknown 2058-2060 special_keyword as -2061-2071 capitalized_identifier SampleLang +2061-2071 type_assertion SampleLang 2072-2081 keyword satisfies -2082-2092 capitalized_identifier SampleLang +2082-2092 type_assertion SampleLang 2092-2093 punctuation ; 2095-2101 special_keyword export 2102-2111 keyword interface -2112-2127 class_name SomeE -2112-2117 capitalized_identifier SomeE +2112-2117 class_name SomeE 2117-2118 operator < -2118-2119 constant T +2118-2119 type_name T 2120-2121 operator = 2122-2126 keyword null 2126-2127 operator > @@ -532,8 +487,7 @@ Token types (27 unique): 2149-2150 operator : 2151-2157 builtin number 2157-2158 punctuation ; -2161-2162 operator ? -2162-2163 operator : +2161-2163 operator ?: 2164-2165 constant T 2165-2166 punctuation ; 2167-2168 punctuation } @@ -553,37 +507,32 @@ Token types (27 unique): 2223-2224 punctuation , 2228-2229 operator : 2230-2233 number 100 -2233-2234 punctuation } -2234-2235 punctuation ; +2233-2235 punctuation }; 2236-2241 keyword const 2244-2245 operator = -2246-2247 punctuation [ -2247-2248 punctuation [ +2246-2248 punctuation [[ 2248-2250 string '' 2250-2251 punctuation , -2253-2254 punctuation ] -2254-2255 punctuation ] +2253-2255 punctuation ]] 2256-2258 special_keyword as 2259-2264 keyword const 2264-2265 punctuation ; 2266-2272 special_keyword export 2273-2278 keyword const -2285-2306 type_annotation : Map +2285-2305 type_annotation : Map 2285-2286 : : -2286-2306 type Map -2287-2290 capitalized_identifier Map +2287-2305 type Map +2287-2290 type_name Map 2290-2291 operator < 2291-2297 builtin string 2297-2298 punctuation , -2299-2304 capitalized_identifier SomeE +2299-2304 type_name SomeE 2304-2305 operator > 2306-2307 operator = 2308-2311 keyword new 2312-2315 class_name Map -2312-2315 capitalized_identifier Map 2315-2316 punctuation ( -2317-2318 punctuation ) -2318-2319 punctuation ; +2317-2319 punctuation ); 2321-2327 special_keyword export 2328-2336 keyword function 2337-2340 function add @@ -612,9 +561,7 @@ Token types (27 unique): 2421-2422 operator : 2423-2426 builtin any 2426-2427 punctuation ) -2427-2433 type_annotation : any -2427-2428 : : -2428-2433 type any +2427-2428 operator : 2429-2432 builtin any 2433-2435 operator => 2438-2439 operator + diff --git a/src/test/fixtures/helpers.ts b/src/test/fixtures/helpers.ts index e9d747f8..3536c46c 100644 --- a/src/test/fixtures/helpers.ts +++ b/src/test/fixtures/helpers.ts @@ -4,6 +4,7 @@ import {basename, join, relative} from 'node:path'; import {syntax_styler_global} from '$lib/syntax_styler_global.ts'; import {tokenize_syntax} from '$lib/tokenize_syntax.ts'; import {type SyntaxTokenStream, SyntaxToken} from '$lib/syntax_token.ts'; +import {syntax_events_to_tokens} from '$lib/lexer.ts'; export interface SampleSpec { lang: string; @@ -128,6 +129,10 @@ const get_token_length = (token: SyntaxToken): number => { * Generate token data from syntax styler */ export const generate_token_data = (sample: SampleSpec): Array => { + // Languages ported to the lexer engine produce tokens from the flat event stream + if (syntax_styler_global.has_lexer_lang(sample.lang)) { + return syntax_events_to_tokens(syntax_styler_global.lex(sample.content, sample.lang)); + } // Get tokens from syntax styler and extract all with positions const grammar = syntax_styler_global.get_lang(sample.lang); const tokens = tokenize_syntax(sample.content, grammar); diff --git a/src/test/lexer.test.ts b/src/test/lexer.test.ts new file mode 100644 index 00000000..2229505f --- /dev/null +++ b/src/test/lexer.test.ts @@ -0,0 +1,178 @@ +import {describe, test, assert} from 'vitest'; + +import { + Lexer, + lex_syntax, + render_syntax_html, + syntax_events_to_tokens, + token_type, + token_type_info, + validate_syntax_events, + type LexedSyntax, + type SyntaxLang, +} from '$lib/lexer.ts'; + +const T_A = token_type('test_a'); +const T_B = token_type('test_b'); +const T_ALIASED = token_type('test_aliased', ['test_alias_1', 'test_alias_2']); +const T_CONTAINER = token_type('test_container'); + +const lexed_of = (text: string, build: (l: Lexer) => void): LexedSyntax => { + const lexer = new Lexer(); + lexer.text = text; + lexer.end = text.length; + build(lexer); + return {text, events: lexer.events, events_len: lexer.events_len}; +}; + +describe('token_type', () => { + test('interns by name and aliases', () => { + assert.strictEqual(token_type('test_a'), T_A); + assert.notStrictEqual(token_type('test_a', 'other'), T_A); + }); + + test('precomputes classes and open tag', () => { + const info = token_type_info(T_ALIASED); + assert.strictEqual(info.classes, 'token_test_aliased token_test_alias_1 token_test_alias_2'); + assert.strictEqual(info.open_tag, ``); + }); +}); + +describe('Lexer', () => { + test('coalesces adjacent same-type leaves', () => { + const lexed = lexed_of('abcdef', (l) => { + l.leaf(T_A, 0, 2); + l.leaf(T_A, 2, 4); + l.leaf(T_B, 4, 5); + }); + assert.deepEqual(syntax_events_to_tokens(lexed), [ + {type: 'test_a', start: 0, end: 4}, + {type: 'test_b', start: 4, end: 5}, + ]); + }); + + test('does not coalesce across gaps or containers', () => { + const lexed = lexed_of('abcdef', (l) => { + l.leaf(T_A, 0, 1); + l.leaf(T_A, 2, 3); // gap between 1 and 2 + l.open(T_CONTAINER, 3); + l.leaf(T_A, 3, 4); + l.close(4); + l.leaf(T_A, 4, 5); // adjacent but separated by a close event + }); + const tokens = syntax_events_to_tokens(lexed); + assert.strictEqual(tokens.length, 5); + }); + + test('drops empty leaves', () => { + const lexed = lexed_of('ab', (l) => { + l.leaf(T_A, 1, 1); + }); + assert.strictEqual(lexed.events_len, 0); + }); + + test('grows the event buffer', () => { + const lexed = lexed_of('x'.repeat(2000), (l) => { + for (let i = 0; i < 1000; i++) { + l.leaf(i % 2 === 0 ? T_A : T_B, i * 2, i * 2 + 2); + } + }); + assert.strictEqual(syntax_events_to_tokens(lexed).length, 1000); + assert.deepEqual(validate_syntax_events(lexed), []); + }); +}); + +describe('render_syntax_html', () => { + test('renders leaves with gaps', () => { + const lexed = lexed_of('a b c', (l) => { + l.leaf(T_A, 0, 1); + l.leaf(T_B, 4, 5); + }); + assert.strictEqual( + render_syntax_html(lexed), + 'a b c', + ); + }); + + test('renders nested containers', () => { + const lexed = lexed_of('abc', (l) => { + l.open(T_CONTAINER, 0); + l.leaf(T_A, 1, 2); + l.close(3); + }); + assert.strictEqual( + render_syntax_html(lexed), + 'abc', + ); + }); + + test('escapes text content', () => { + const lexed = lexed_of('a &  c', (l) => { + l.leaf(T_A, 4, 7); + }); + assert.strictEqual( + render_syntax_html(lexed), + 'a & <b> c', + ); + }); +}); + +describe('validate_syntax_events', () => { + test('flags overlapping leaves', () => { + const lexed = lexed_of('abcdef', (l) => { + l.leaf(T_A, 0, 3); + l.leaf(T_B, 2, 4); + }); + assert.isNotEmpty(validate_syntax_events(lexed)); + }); + + test('flags unclosed containers', () => { + const lexed = lexed_of('abc', (l) => { + l.open(T_CONTAINER, 0); + }); + assert.isNotEmpty(validate_syntax_events(lexed)); + }); + + test('accepts a valid stream', () => { + const lexed = lexed_of('abcdef', (l) => { + l.leaf(T_A, 0, 2); + l.open(T_CONTAINER, 2); + l.leaf(T_B, 2, 3); + l.leaf(T_A, 3, 4); + l.close(5); + l.leaf(T_B, 5, 6); + }); + assert.deepEqual(validate_syntax_events(lexed), []); + }); +}); + +describe('embed', () => { + test('lexes a window with another registered language', () => { + const inner: SyntaxLang = { + id: 'test_inner', + lex: (l) => { + l.leaf(T_B, l.pos, l.end); + l.pos = l.end; + }, + }; + const outer: SyntaxLang = { + id: 'test_outer', + lex: (l) => { + l.open(T_CONTAINER, 0); + l.embed('test_inner', 1, 3); + l.close(4); + l.pos = l.end; + }, + }; + const langs = new Map([ + ['test_inner', inner], + ['test_outer', outer], + ]); + const lexed = lex_syntax('abcd', outer, langs); + assert.deepEqual(validate_syntax_events(lexed), []); + assert.deepEqual(syntax_events_to_tokens(lexed), [ + {type: 'test_container', start: 0, end: 4}, + {type: 'test_b', start: 1, end: 3}, + ]); + }); +}); diff --git a/src/test/lexer_bash.test.ts b/src/test/lexer_bash.test.ts new file mode 100644 index 00000000..5ba13dbe --- /dev/null +++ b/src/test/lexer_bash.test.ts @@ -0,0 +1,215 @@ +import {describe, test, assert} from 'vitest'; +import {readFileSync} from 'node:fs'; + +import {syntax_styler_global} from '$lib/syntax_styler_global.ts'; +import {syntax_events_to_tokens, validate_syntax_events} from '$lib/lexer.ts'; + +const tokens_of = (text: string): Array<[string, string]> => + syntax_events_to_tokens(syntax_styler_global.lex(text, 'bash')).map((t) => [ + t.type, + text.slice(t.start, t.end), + ]); + +const picked = (text: string, types: Array): Array<[string, string]> => + tokens_of(text).filter(([type]) => types.includes(type)); + +describe('lexer_bash words', () => { + test('keywords, builtins, booleans by lookup', () => { + assert.deepEqual(picked('if true; then echo hi; fi', ['keyword', 'builtin', 'boolean']), [ + ['keyword', 'if'], + ['boolean', 'true'], + ['keyword', 'then'], + ['builtin', 'echo'], + ['keyword', 'fi'], + ]); + }); + + test('word boundaries — local_var is not the local keyword', () => { + assert.deepEqual(picked('local_var=1', ['keyword']), []); + assert.deepEqual(picked('local x=1', ['keyword']), [['keyword', 'local']]); + }); + + test('function definitions, both styles', () => { + assert.deepEqual(picked('greet() { :; }', ['function']), [['function', 'greet']]); + assert.deepEqual(picked('function cleanup { :; }', ['keyword', 'function']), [ + ['keyword', 'function'], + ['function', 'cleanup'], + ]); + }); +}); + +describe('lexer_bash variables and expansions', () => { + test('special, braced, and named variables', () => { + assert.deepEqual(picked('echo $0 $@ ${name} ${arr[0]} $HOME', ['variable']), [ + ['variable', '$0'], + ['variable', '$@'], + ['variable', '${name}'], + ['variable', '${arr[0]}'], + ['variable', '$HOME'], + ]); + }); + + test('command substitution is a container with nested bash', () => { + assert.deepEqual(tokens_of('$(echo hi)'), [ + ['command_substitution', '$(echo hi)'], + ['punctuation', '$('], + ['builtin', 'echo'], + ['punctuation', ')'], + ]); + }); + + test('nested command substitution inside a double-quoted string', () => { + const lexed = syntax_styler_global.lex('"a $(echo "b $(echo c)")"', 'bash'); + assert.deepEqual(validate_syntax_events(lexed), []); + const types = syntax_events_to_tokens(lexed).map((t) => t.type); + assert.strictEqual(types.filter((t) => t === 'command_substitution').length, 2); + assert.strictEqual(types.filter((t) => t === 'string').length, 2); + }); + + test('double-quoted strings expand, single-quoted do not', () => { + assert.deepEqual(picked('"$a" \'$a\'', ['string', 'variable']), [ + ['string', '"$a"'], + ['variable', '$a'], + ['string', "'$a'"], + ]); + }); + + test('a bare $ before a non-name char is plain', () => { + assert.deepEqual(picked('"x$"', ['variable']), []); + }); +}); + +describe('lexer_bash arithmetic', () => { + test('$((…)) is distinct from command substitution', () => { + const tokens = tokens_of('$(( 2 + 3 ))'); + assert.deepEqual(picked('$(( 2 + 3 ))', ['command_substitution']), []); + assert.deepEqual(tokens, [ + ['punctuation', '$(('], + ['number', '2'], + ['number', '3'], + ['punctuation', '))'], + ]); + }); + + test('$(…) command substitution still parses', () => { + assert.deepEqual(picked('$(date)', ['command_substitution']), [ + ['command_substitution', '$(date)'], + ]); + }); +}); + +describe('lexer_bash numbers, operators, descriptors', () => { + test('number formats', () => { + assert.deepEqual(picked('x=0xFF y=077 z=2#1010 w=42', ['number']), [ + ['number', '0xFF'], + ['number', '077'], + ['number', '2#1010'], + ['number', '42'], + ]); + }); + + test('operators and here-string vs heredoc', () => { + assert.deepEqual(picked('a || b && c | d <<< e', ['operator']), [ + ['operator', '||'], + ['operator', '&&'], + ['operator', '|'], + ['operator', '<<<'], + ]); + }); + + test('file descriptors around redirections', () => { + assert.deepEqual(picked('cmd 2>&1', ['file_descriptor', 'operator']), [ + ['file_descriptor', '2'], + ['operator', '>'], + ['file_descriptor', '&1'], + ]); + }); + + test('bare = is plain, but == and =~ are operators', () => { + assert.deepEqual(picked('x=1; [[ $x == 1 ]]; [[ $x =~ a ]]', ['operator']), [ + ['operator', '=='], + ['operator', '=~'], + ]); + }); +}); + +describe('lexer_bash heredocs', () => { + test('unquoted heredoc is a container that expands variables', () => { + const text = 'cat < { + const text = "cat <<'END'\nno $expansion\nEND\n"; + assert.deepEqual(picked(text, ['variable', 'heredoc_delimiter']), [ + ['heredoc_delimiter', "<<'END'"], + ['heredoc_delimiter', 'END'], + ]); + }); + + test('<<- allows an indented closing delimiter', () => { + const text = 'cat <<-EOF\n\tbody\n\tEOF\n'; + const tokens = tokens_of(text); + assert.ok(tokens.some(([type, t]) => type === 'heredoc_delimiter' && t === 'EOF')); + assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(text, 'bash')), []); + }); + + test('unterminated heredoc extends to the window end without throwing', () => { + const lexed = syntax_styler_global.lex('cat < { + const text = 'cat < t.type === 'heredoc_delimiter' && text.slice(t.start, t.end) === 'EOF'), + ); + assert.ok(tokens.some((t) => t.type === 'variable')); + }); +}); + +describe('lexer_bash comments', () => { + test('comments require a preceding boundary; $# is a variable', () => { + assert.deepEqual(picked('echo $# # trailing', ['comment', 'variable']), [ + ['variable', '$#'], + ['comment', '# trailing'], + ]); + }); + + test('shebang at file start', () => { + assert.deepEqual(tokens_of('#!/bin/bash\nx')[0], ['shebang', '#!/bin/bash']); + }); +}); + +describe('lexer_bash sample', () => { + const content = readFileSync('src/test/fixtures/samples/sample_complex.bash', 'utf8'); + + test('sample lexes with valid invariants', () => { + const lexed = syntax_styler_global.lex(content, 'bash'); + assert.deepEqual(validate_syntax_events(lexed), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 11) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'bash'); + assert.deepEqual(validate_syntax_events(lexed), [], `prefix of length ${len}`); + } + }); + + test('lexing is deterministic', () => { + const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'bash')); + const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'bash')); + assert.deepEqual(a, b); + }); +}); diff --git a/src/test/lexer_css.test.ts b/src/test/lexer_css.test.ts new file mode 100644 index 00000000..0f8d88f8 --- /dev/null +++ b/src/test/lexer_css.test.ts @@ -0,0 +1,146 @@ +import {describe, test, assert} from 'vitest'; +import {readFileSync} from 'node:fs'; + +import {syntax_styler_global} from '$lib/syntax_styler_global.ts'; +import {syntax_events_to_tokens, validate_syntax_events} from '$lib/lexer.ts'; + +const tokens_of = (text: string): Array<[string, string]> => + syntax_events_to_tokens(syntax_styler_global.lex(text, 'css')).map((t) => [ + t.type, + text.slice(t.start, t.end), + ]); + +const picked = (text: string, types: Array): Array<[string, string]> => + tokens_of(text).filter(([type]) => types.includes(type)); + +describe('lexer_css structure', () => { + test('a simple rule: selector, property, value stays plain', () => { + assert.deepEqual(tokens_of('.a { color: red; }'), [ + ['selector', '.a'], + ['punctuation', '{'], + ['property', 'color'], + ['punctuation', ':'], + ['punctuation', ';'], + ['punctuation', '}'], + ]); + }); + + test('selectors are a single leaf, including pseudo and attribute parts', () => { + assert.deepEqual(picked('.content::before { x: 1 }', ['selector']), [ + ['selector', '.content::before'], + ]); + assert.deepEqual(picked("a[title='Click: here'] { x: 1 }", ['selector']), [ + ['selector', "a[title='Click: here']"], + ]); + assert.deepEqual(picked('div > p { x: 1 }', ['selector']), [['selector', 'div > p']]); + }); + + test('native nesting: a nested rule inside a declaration block', () => { + assert.deepEqual(picked('.a { color: red; .b { color: blue; } }', ['selector', 'property']), [ + ['selector', '.a'], + ['property', 'color'], + ['selector', '.b'], + ['property', 'color'], + ]); + }); +}); + +describe('lexer_css comments and strings', () => { + test('comments win over their contents', () => { + assert.deepEqual(picked('/* .a { x: 1 } */ .b {}', ['comment', 'selector']), [ + ['comment', '/* .a { x: 1 } */'], + ['selector', '.b'], + ]); + }); + + test('a string is not broken by an inner comment sequence', () => { + assert.deepEqual(picked('.a { content: "/* x */"; }', ['string', 'comment']), [ + ['string', '"/* x */"'], + ]); + }); + + test('unterminated comment extends to end', () => { + const lexed = syntax_styler_global.lex('.a { /* unterminated', 'css'); + assert.deepEqual(validate_syntax_events(lexed), []); + }); +}); + +describe('lexer_css values', () => { + test('functions and url() containers', () => { + assert.deepEqual(tokens_of('.a { background: url("x.png"); }').slice(2), [ + ['property', 'background'], + ['punctuation', ':'], + ['url', 'url("x.png")'], + ['function', 'url'], + ['punctuation', '('], + ['string', '"x.png"'], + ['punctuation', ')'], + ['punctuation', ';'], + ['punctuation', '}'], + ]); + }); + + test('function calls in values', () => { + assert.deepEqual(picked('.a { color: rgba(0, 0, 0, 0.1); }', ['function', 'property']), [ + ['property', 'color'], + ['function', 'rgba'], + ]); + }); + + test('!important', () => { + assert.deepEqual(picked('.a { color: red !important; }', ['important']), [ + ['important', '!important'], + ]); + }); + + test('numbers and bare identifiers in values stay plain (parity)', () => { + assert.deepEqual(picked('.a { margin: 10px; color: red; }', ['number', 'property']), [ + ['property', 'margin'], + ['property', 'color'], + ]); + }); +}); + +describe('lexer_css at-rules', () => { + test('@media prelude with a feature is an atrule container', () => { + const tokens = tokens_of('@media (max-width: 600px) { body {} }'); + assert.deepEqual(tokens.slice(0, 6), [ + ['atrule', '@media (max-width: 600px)'], + ['rule', '@media'], + ['punctuation', '('], + ['property', 'max-width'], + ['punctuation', ':'], + ['punctuation', ')'], + ]); + }); + + test('at-rule prelude keywords', () => { + assert.deepEqual(picked('@media screen and (min-width: 1px) {}', ['keyword', 'rule']), [ + ['rule', '@media'], + ['keyword', 'and'], + ]); + }); +}); + +describe('lexer_css sample', () => { + const content = readFileSync('src/test/fixtures/samples/sample_complex.css', 'utf8'); + + test('sample lexes with valid invariants', () => { + const lexed = syntax_styler_global.lex(content, 'css'); + assert.deepEqual(validate_syntax_events(lexed), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 5) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'css'); + assert.deepEqual(validate_syntax_events(lexed), [], `prefix of length ${len}`); + } + }); + + test('lexing is deterministic', () => { + const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'css')); + const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'css')); + assert.deepEqual(a, b); + }); +}); diff --git a/src/test/lexer_json.test.ts b/src/test/lexer_json.test.ts new file mode 100644 index 00000000..1943e32f --- /dev/null +++ b/src/test/lexer_json.test.ts @@ -0,0 +1,92 @@ +import {describe, test, assert} from 'vitest'; +import {readFileSync} from 'node:fs'; + +import {syntax_styler_global} from '$lib/syntax_styler_global.ts'; +import {syntax_events_to_tokens, validate_syntax_events} from '$lib/lexer.ts'; + +const tokens_of = (text: string): Array<[string, string]> => + syntax_events_to_tokens(syntax_styler_global.lex(text, 'json')).map((t) => [ + t.type, + text.slice(t.start, t.end), + ]); + +describe('lexer_json', () => { + test('distinguishes properties from string values', () => { + assert.deepEqual(tokens_of('{"a": "b"}'), [ + ['punctuation', '{'], + ['property', '"a"'], + ['operator', ':'], + ['string', '"b"'], + ['punctuation', '}'], + ]); + }); + + test('handles literals and numbers', () => { + assert.deepEqual(tokens_of('[true, false, null, -1.5e3]'), [ + ['punctuation', '['], + ['boolean', 'true'], + ['punctuation', ','], + ['boolean', 'false'], + ['punctuation', ','], + ['null', 'null'], + ['punctuation', ','], + ['number', '-1.5e3'], + ['punctuation', ']'], + ]); + }); + + test('null renders with the keyword alias class', () => { + assert.include( + syntax_styler_global.stylize('null', 'json'), + 'null', + ); + }); + + test('coalesces adjacent punctuation', () => { + assert.deepEqual(tokens_of('[[]]'), [['punctuation', '[[]]']]); + }); + + test('handles jsonc comments', () => { + assert.deepEqual(tokens_of('// line\n{/* block */}'), [ + ['comment', '// line'], + ['punctuation', '{'], + ['comment', '/* block */'], + ['punctuation', '}'], + ]); + }); + + test('unterminated string extends to end of line', () => { + assert.deepEqual(tokens_of('"abc\n1'), [ + ['string', '"abc'], + ['number', '1'], + ]); + }); + + test('property lookahead crosses newlines', () => { + assert.deepEqual(tokens_of('{"a"\n: 1}'), [ + ['punctuation', '{'], + ['property', '"a"'], + ['operator', ':'], + ['number', '1'], + ['punctuation', '}'], + ]); + }); +}); + +describe('lexer_json sample', () => { + const content = readFileSync('src/test/fixtures/samples/sample_complex.json', 'utf8'); + + test('sample lexes with valid invariants', () => { + const lexed = syntax_styler_global.lex(content, 'json'); + assert.deepEqual(validate_syntax_events(lexed), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 7) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'json'); + const issues = validate_syntax_events(lexed); + assert.deepEqual(issues, [], `prefix of length ${len}`); + } + }); +}); diff --git a/src/test/lexer_ts.test.ts b/src/test/lexer_ts.test.ts new file mode 100644 index 00000000..94684fda --- /dev/null +++ b/src/test/lexer_ts.test.ts @@ -0,0 +1,273 @@ +import {describe, test, assert} from 'vitest'; +import {readFileSync} from 'node:fs'; + +import {syntax_styler_global} from '$lib/syntax_styler_global.ts'; +import {syntax_events_to_tokens, validate_syntax_events} from '$lib/lexer.ts'; + +/** + * Lexes `text` as ts and returns `[type, text]` pairs. + * Containers appear before their children with their full span text. + */ +const tokens_of = (text: string): Array<[string, string]> => + syntax_events_to_tokens(syntax_styler_global.lex(text, 'ts')).map((t) => [ + t.type, + text.slice(t.start, t.end), + ]); + +/** + * Like `tokens_of` but only `[type, text]` of the named types — for targeted + * assertions that ignore surrounding punctuation/operators. + */ +const picked = (text: string, types: Array): Array<[string, string]> => + tokens_of(text).filter(([type]) => types.includes(type)); + +describe('lexer_ts keywords', () => { + test('main and special keywords', () => { + assert.deepEqual(picked('if (x) return new Foo();', ['keyword', 'special_keyword']), [ + ['special_keyword', 'if'], + ['special_keyword', 'return'], + ['keyword', 'new'], + ]); + }); + + test('keywords after member access are properties, not keywords', () => { + assert.deepEqual(picked('a.delete(); b?.new;', ['keyword', 'special_keyword']), []); + // and `a.delete()` is a function call + assert.deepEqual(picked('a.delete();', ['function']), [['function', 'delete']]); + }); + + test('contextual keywords', () => { + assert.deepEqual(picked('async () => x; const get = 1; get x() {}', ['keyword']), [ + ['keyword', 'async'], + ['keyword', 'const'], + ['keyword', 'get'], + ]); + }); + + test('null and undefined are keywords, true/false booleans, NaN a number', () => { + assert.deepEqual(tokens_of('null undefined true NaN'), [ + ['keyword', 'null'], + ['keyword', 'undefined'], + ['boolean', 'true'], + ['number', 'NaN'], + ]); + }); +}); + +describe('lexer_ts strings and templates', () => { + test('string properties in object literals', () => { + assert.deepEqual(picked('{"a": 1, \'b\': 2}', ['string_property', 'string']), [ + ['string_property', '"a"'], + ['string_property', "'b'"], + ]); + }); + + test('ternary strings are not properties', () => { + assert.deepEqual(picked('x ? "a" : "b"', ['string_property', 'string']), [ + ['string', '"a"'], + ['string', '"b"'], + ]); + }); + + test('strings containing // are not comments', () => { + assert.deepEqual(picked('const a = "x // y";', ['string', 'comment']), [ + ['string', '"x // y"'], + ]); + }); + + test('template strings with interpolation', () => { + const tokens = tokens_of('`a${b}c`'); + assert.deepEqual(tokens, [ + ['template_string', '`a${b}c`'], + ['template_punctuation', '`'], + ['string', 'a'], + ['interpolation', '${b}'], + ['interpolation_punctuation', '${'], + ['interpolation_punctuation', '}'], + ['string', 'c'], + ['template_punctuation', '`'], + ]); + }); + + test('nested templates in interpolations', () => { + const text = '`a${`b${c}`}d`'; + const lexed = syntax_styler_global.lex(text, 'ts'); + assert.deepEqual(validate_syntax_events(lexed), []); + const types = syntax_events_to_tokens(lexed).map((t) => t.type); + assert.strictEqual(types.filter((t) => t === 'template_string').length, 2); + }); + + test('unterminated template extends to end', () => { + const lexed = syntax_styler_global.lex('`abc', 'ts'); + assert.deepEqual(validate_syntax_events(lexed), []); + const tokens = syntax_events_to_tokens(lexed); + assert.strictEqual(tokens[0]!.type, 'template_string'); + assert.strictEqual(tokens[0]!.end, 4); + }); +}); + +describe('lexer_ts regex vs division', () => { + test('regex literal in expression position', () => { + assert.deepEqual(picked('const re = /ab+c/gi;', ['regex', 'regex_source', 'regex_flags']), [ + ['regex', '/ab+c/gi'], + ['regex_source', 'ab+c'], + ['regex_flags', 'gi'], + ]); + }); + + test('division is not a regex', () => { + assert.deepEqual(picked('const x = a / b / c;', ['regex']), []); + }); + + test('regex after return keyword', () => { + assert.deepEqual(picked('return /x/;', ['regex']), [['regex', '/x/']]); + }); + + test('regex with slash in character class', () => { + assert.deepEqual(picked('x = /[/]/;', ['regex']), [['regex', '/[/]/']]); + }); +}); + +describe('lexer_ts identifiers', () => { + test('class contexts', () => { + assert.deepEqual(picked('class Foo extends Bar {}', ['class_name']), [ + ['class_name', 'Foo'], + ['class_name', 'Bar'], + ]); + assert.deepEqual(picked('new a.b.Thing()', ['class_name']), [ + ['class_name', 'a'], + ['class_name', 'b'], + ['class_name', 'Thing'], + ]); + }); + + test('constants and capitalized identifiers', () => { + assert.deepEqual(tokens_of('MAX_VALUE Foo'), [ + ['constant', 'MAX_VALUE'], + ['capitalized_identifier', 'Foo'], + ]); + }); + + test('function calls and function-valued variables', () => { + assert.deepEqual(picked('foo(); const f = (a) => a;', ['function', 'function_variable']), [ + ['function', 'foo'], + ['function_variable', 'f'], + ]); + }); + + test('generic function calls', () => { + const tokens = tokens_of('foo(x)'); + assert.deepEqual(tokens[0], ['generic_function', 'foo']); + assert.deepEqual(picked('foo(x)', ['function', 'type_name']), [ + ['function', 'foo'], + ['type_name', 'Bar'], + ]); + }); + + test('comparisons are not generics', () => { + assert.deepEqual(picked('a < b; c > d;', ['generic_function']), []); + assert.deepEqual(picked('a < b; c > d;', ['operator']), [ + ['operator', '<'], + ['operator', '>'], + ]); + }); + + test('lowercase builtins', () => { + assert.deepEqual(picked('console.log(x); const s: unknown = 1;', ['builtin']), [ + ['builtin', 'console'], + ['builtin', 'unknown'], // inside the type annotation region + ]); + }); +}); + +describe('lexer_ts type syntax', () => { + test('type alias declarations', () => { + assert.deepEqual(picked('type Foo = Bar;', ['keyword', 'class_name']), [ + ['keyword', 'type'], + ['class_name', 'Foo'], + ]); + }); + + test('import type', () => { + assert.deepEqual(picked("import type {A} from 'b';", ['import_type_keyword']), [ + ['import_type_keyword', 'type'], + ]); + }); + + test('type assertions with as and satisfies', () => { + assert.deepEqual(picked('x as Foo; y satisfies Baz;', ['type_assertion']), [ + ['type_assertion', 'Foo'], + ['type_assertion', 'Baz'], + ]); + }); + + test('type annotations with initializers', () => { + const tokens = tokens_of('const x: Map = y;'); + const annotation = tokens.find(([type]) => type === 'type_annotation'); + assert.ok(annotation); + assert.deepEqual(picked('const x: Map = y;', ['type_name']), [ + ['type_name', 'Map'], + ['type_name', 'Foo'], + ]); + }); + + test('object literal colons are not annotations', () => { + assert.deepEqual(picked('const o = {a: b};', ['type_annotation']), []); + }); +}); + +describe('lexer_ts decorators and misc', () => { + test('decorators', () => { + assert.deepEqual(tokens_of('@component'), [ + ['decorator', '@component'], + ['at', '@'], + ['function', 'component'], + ]); + }); + + test('hashbang', () => { + assert.deepEqual(tokens_of('#!/usr/bin/env node\nx')[0], ['hashbang', '#!/usr/bin/env node']); + }); + + test('numeric formats', () => { + assert.deepEqual(picked('0xff 0b10 1_000n 1.5e-3 .5', ['number']), [ + ['number', '0xff'], + ['number', '0b10'], + ['number', '1_000n'], + ['number', '1.5e-3'], + ['number', '.5'], + ]); + }); + + test('spread and optional chaining operators', () => { + assert.deepEqual(picked('f(...a); b?.c ?? d;', ['operator']), [ + ['operator', '...'], + ['operator', '?.'], + ['operator', '??'], + ]); + }); +}); + +describe('lexer_ts sample', () => { + const content = readFileSync('src/test/fixtures/samples/sample_complex.ts', 'utf8'); + + test('sample lexes with valid invariants', () => { + const lexed = syntax_styler_global.lex(content, 'ts'); + assert.deepEqual(validate_syntax_events(lexed), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 13) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'ts'); + const issues = validate_syntax_events(lexed); + assert.deepEqual(issues, [], `prefix of length ${len}`); + } + }); + + test('lexing is deterministic', () => { + const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'ts')); + const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'ts')); + assert.deepEqual(a, b); + }); +}); From 8e0a984044b35eb86119af06c4544ae10f4d9e2d Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 11:09:45 -0400 Subject: [PATCH 02/25] cleanup --- src/lib/lexer.ts | 10 +++--- src/lib/lexer_ts.ts | 84 ++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/src/lib/lexer.ts b/src/lib/lexer.ts index 4c3e3036..8cb0613b 100644 --- a/src/lib/lexer.ts +++ b/src/lib/lexer.ts @@ -188,6 +188,9 @@ export class Lexer { const prev_end = this.end; this.pos = start; this.end = end; + // reset coalescing state so the guest's first leaf can't merge with a + // leaf the host emitted just before the embedded region + this.#last_leaf = -1; lang.lex(this); this.pos = prev_pos; this.end = prev_end; @@ -380,12 +383,11 @@ export const validate_syntax_events = (lexed: LexedSyntax): Array => { // policy (matches the inherited U+00A0-U+FFFF identifier ranges; surrogate // halves are >= 0xa0 so astral chars behave consistently). export const CF_SPACE = 1; -export const CF_DIGIT = 2; -export const CF_IDENT_START = 4; -export const CF_IDENT = 8; +export const CF_IDENT_START = 2; +export const CF_IDENT = 4; export const CHAR_FLAGS: Uint8Array = new Uint8Array(128); -for (let c = 48; c <= 57; c++) CHAR_FLAGS[c] = CF_DIGIT | CF_IDENT; // 0-9 +for (let c = 48; c <= 57; c++) CHAR_FLAGS[c] = CF_IDENT; // 0-9 for (let c = 65; c <= 90; c++) CHAR_FLAGS[c] = CF_IDENT_START | CF_IDENT; // A-Z for (let c = 97; c <= 122; c++) CHAR_FLAGS[c] = CF_IDENT_START | CF_IDENT; // a-z CHAR_FLAGS[36] = CF_IDENT_START | CF_IDENT; // $ diff --git a/src/lib/lexer_ts.ts b/src/lib/lexer_ts.ts index 434ff486..94ad020c 100644 --- a/src/lib/lexer_ts.ts +++ b/src/lib/lexer_ts.ts @@ -482,7 +482,7 @@ const scan_operator = (text: string, i: number, end: number): number => { * Lexes a template literal starting at the backtick at `i`, returning the new * position. Interpolations recurse through the full lexer. */ -const lex_ts_template = (l: Lexer, i: number, to: number, js: boolean): number => { +const lex_ts_template = (l: Lexer, i: number, to: number): number => { const {text} = l; l.open(T_TEMPLATE_STRING, i); l.leaf(T_TEMPLATE_PUNCTUATION, i, i + 1); @@ -505,7 +505,7 @@ const lex_ts_template = (l: Lexer, i: number, to: number, js: boolean): number = const inner_end = close === -1 ? to : close; l.open(T_INTERPOLATION, j); l.leaf(T_INTERPOLATION_PUNCTUATION, j, j + 2); - lex_ts_window(l, j + 2, inner_end, false, js); + lex_ts_window(l, j + 2, inner_end, false); if (close === -1) { l.close(to); j = to; @@ -531,13 +531,7 @@ const lex_ts_template = (l: Lexer, i: number, to: number, js: boolean): number = * The core window lexer. `type_mode` switches capitalized identifiers to * `type_name` (used for generics and type annotations). */ -const lex_ts_window = ( - l: Lexer, - from: number, - to: number, - type_mode: boolean, - js: boolean, -): void => { +const lex_ts_window = (l: Lexer, from: number, to: number, type_mode: boolean): void => { const {text} = l; let i = from; let prev = P_NONE; @@ -572,26 +566,24 @@ const lex_ts_window = ( if (was_class_ctx && kind === undefined && c !== 35) { // class-name chain: `Foo`, `a.b.Foo`, optionally with generics l.leaf(T_CLASS_NAME, start, ident_end); - while (text.charCodeAt(i) === 46 && is_ident_start(text.charCodeAt(i + 1)) && i + 1 < to) { + while (i + 1 < to && text.charCodeAt(i) === 46 && is_ident_start(text.charCodeAt(i + 1))) { l.leaf(T_PUNCTUATION, i, i + 1); const seg_end = scan_ident(text, i + 1, to); l.leaf(T_CLASS_NAME, i + 1, seg_end); i = seg_end; } - if (!js) { - const angle_start = skip_space(text, i, to); - if (text.charCodeAt(angle_start) === 60) { - const angle_end = scan_balanced_angle(text, angle_start, to); - if (angle_end !== -1) { - lex_ts_window(l, angle_start, angle_end + 1, true, js); - i = angle_end + 1; - } + const angle_start = skip_space(text, i, to); + if (text.charCodeAt(angle_start) === 60) { + const angle_end = scan_balanced_angle(text, angle_start, to); + if (angle_end !== -1) { + lex_ts_window(l, angle_start, angle_end + 1, true); + i = angle_end + 1; } } continue; } - if (was_as_ctx && kind === undefined && c !== 35 && !js && !BUILTIN_WORDS.has(word)) { + if (was_as_ctx && kind === undefined && c !== 35 && !BUILTIN_WORDS.has(word)) { // `x as Foo` — but `as unknown`/`as string` keep their builtin type l.leaf(T_TYPE_ASSERTION, start, ident_end); continue; @@ -608,7 +600,7 @@ const lex_ts_window = ( keyword_id = T_SPECIAL_KEYWORD; break; case K_TS: - if (!js) keyword_id = T_KEYWORD; + keyword_id = T_KEYWORD; break; case K_ASYNC: { const n = text.charCodeAt(skip_space(text, ident_end, to)); @@ -631,7 +623,6 @@ const lex_ts_window = ( break; } case K_TYPE_WORD: { - if (js) break; const n = text.charCodeAt(skip_space(text, ident_end, to)); // `import type {…}` / `export type * from …` — a type-only // import/export modifier, not a type-alias declaration @@ -645,7 +636,6 @@ const lex_ts_window = ( break; } case K_TS_COND: { - if (js) break; const n = text.charCodeAt(skip_space(text, ident_end, to)); if (n === 123 || is_ident_start(n) || Number.isNaN(n)) { keyword_id = T_KEYWORD; @@ -661,8 +651,8 @@ const lex_ts_window = ( } if (keyword_id !== 0) { l.leaf(keyword_id, start, ident_end); - if (CLASS_CTX_WORDS.has(word) && (word !== 'type' || !js)) class_ctx = true; - if (AS_WORDS.has(word) && !js) as_ctx = true; + if (CLASS_CTX_WORDS.has(word)) class_ctx = true; + if (AS_WORDS.has(word)) as_ctx = true; if (IMPORT_WORDS.has(word)) import_ctx = true; if (!VALUE_WORDS.has(word)) prev = P_NONE; continue; @@ -703,7 +693,7 @@ const lex_ts_window = ( } // generic call: `foo(…)` - if (!js) { + { const angle_start = skip_space(text, ident_end, to); if (text.charCodeAt(angle_start) === 60) { const angle_end = scan_balanced_angle(text, angle_start, to); @@ -711,7 +701,7 @@ const lex_ts_window = ( l.open(T_GENERIC_FUNCTION, start); l.leaf(T_FUNCTION, start, ident_end); l.open(T_GENERIC, angle_start); - lex_ts_window(l, angle_start, angle_end + 1, true, js); + lex_ts_window(l, angle_start, angle_end + 1, true); l.close(angle_end + 1); l.close(angle_end + 1); i = angle_end + 1; @@ -801,7 +791,7 @@ const lex_ts_window = ( // template literals if (c === 96) { - i = lex_ts_template(l, i, to, js); + i = lex_ts_template(l, i, to); prev = P_VALUE; prev_code = 0; class_ctx = as_ctx = import_ctx = false; @@ -853,7 +843,7 @@ const lex_ts_window = ( // `:` — type annotation (with initializer) or plain operator if (c === 58) { class_ctx = as_ctx = import_ctx = false; - if (!type_mode && !js) { + if (!type_mode) { const type_end = scan_type_annotation(text, i, to); if (type_end !== -1) { const type_start = skip_space(text, i + 1, to); @@ -865,7 +855,7 @@ const lex_ts_window = ( l.open(T_TYPE_ANNOTATION, i); l.leaf(T_COLON, i, i + 1); l.open(T_TYPE, type_start); - lex_ts_window(l, type_start, content_end, true, js); + lex_ts_window(l, type_start, content_end, true); l.close(content_end); l.close(content_end); i = type_end; @@ -968,21 +958,29 @@ const scan_regex_body = (text: string, i: number, to: number): number => { return -1; }; -const create_lex_ts = - (js: boolean) => - (l: Lexer): void => { - let i = l.pos; - // hashbang at the very start of the document - if (i === 0 && l.text.charCodeAt(0) === 35 && l.text.charCodeAt(1) === 33) { - const line_end = scan_to_line_end(l.text, 0, l.end); - l.leaf(T_HASHBANG, 0, line_end); - i = line_end; - } - lex_ts_window(l, i, l.end, false, js); - l.pos = l.end; - }; +const lex_ts = (l: Lexer): void => { + let i = l.pos; + // hashbang at the very start of the document + if (i === 0 && l.text.charCodeAt(0) === 35 && l.text.charCodeAt(1) === 33) { + const line_end = scan_to_line_end(l.text, 0, l.end); + l.leaf(T_HASHBANG, 0, line_end); + i = line_end; + } + lex_ts_window(l, i, l.end, false); + l.pos = l.end; +}; /** * The TypeScript language registration for the lexer engine. + * + * JavaScript reuses this lexer via the `js`/`javascript` aliases: TypeScript is + * a syntactic superset, and the TS-only constructs the lexer recognizes (type + * annotations, `as`, generics before a call) can't appear in valid JS, so + * running the full TS lexer on JS is a no-op for those paths. There is no + * separate JS lexer. */ -export const lexer_ts: SyntaxLang = {id: 'ts', aliases: ['typescript'], lex: create_lex_ts(false)}; +export const lexer_ts: SyntaxLang = { + id: 'ts', + aliases: ['typescript', 'js', 'javascript'], + lex: lex_ts, +}; From 8eac73965255910f7028416de2ffabe292c9becc Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 12:15:00 -0400 Subject: [PATCH 03/25] wip --- CLAUDE.md | 53 +- benchmark/benchmarks.ts | 29 +- src/lib/lexer.ts | 107 ++-- src/lib/lexer_markup.ts | 517 ++++++++++++++++++ src/lib/lexer_ts.ts | 83 ++- src/lib/syntax_styler.ts | 47 +- src/lib/syntax_styler_global.ts | 3 + .../fixtures/generated/html/html_complex.html | 2 +- .../fixtures/generated/html/html_complex.txt | 6 +- src/test/lexer.pathological.test.ts | 61 +++ src/test/lexer.test.ts | 33 +- src/test/lexer_markup.test.ts | 313 +++++++++++ src/test/pathological.ts | 132 +++++ src/test/syntax_styler.test.ts | 28 + 14 files changed, 1329 insertions(+), 85 deletions(-) create mode 100644 src/lib/lexer_markup.ts create mode 100644 src/test/lexer.pathological.test.ts create mode 100644 src/test/lexer_markup.test.ts create mode 100644 src/test/pathological.ts diff --git a/CLAUDE.md b/CLAUDE.md index b4648d77..7aed41a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -67,9 +67,11 @@ src/ ├── lib/ # exportable library code │ ├── syntax_styler.ts # SyntaxStyler class, hook system │ ├── syntax_styler_global.ts # pre-configured global instance -│ ├── tokenize_syntax.ts # tokenize_syntax() function +│ ├── lexer.ts # lexer-engine substrate: Lexer, TokenTypeRegistry, flat events, HTML render +│ ├── lexer_*.ts # hand-written lexers (json, ts, css, bash, markup) +│ ├── tokenize_syntax.ts # tokenize_syntax() function (regex engine) │ ├── syntax_token.ts # SyntaxToken class, type definitions -│ ├── grammar_*.ts # language definitions (8 files) +│ ├── grammar_*.ts # regex language definitions (8 files) │ ├── Code.svelte # main Svelte component │ ├── CodeHighlight.svelte # experimental CSS Highlight API │ ├── highlight_manager.ts # CSS Highlight API manager @@ -80,6 +82,9 @@ src/ ├── test/ # test files and fixtures │ ├── syntax_styler.test.ts │ ├── highlight_manager.test.ts +│ ├── lexer*.test.ts # lexer-engine suites (substrate + per language) +│ ├── lexer.pathological.test.ts # linearity + validity on adversarial inputs +│ ├── pathological.ts # pathological input generators (tests + benchmark) │ └── fixtures/ │ ├── samples/ # source of truth sample files │ ├── generated/ # generated fixture outputs @@ -93,15 +98,24 @@ src/ ### Core system -**SyntaxStyler** - The main class for tokenization and HTML generation. Uses -regex-based tokenization inherited from PrismJS, maintaining compatibility with -existing language definitions. +Two engines currently coexist (the lexer engine is replacing the regex +engine, language by language, on the `lexer-architecture` branch): -**syntax_styler_global** - Pre-configured instance with all built-in grammars -registered. Import this for typical usage. +**Lexer engine** (`lexer.ts` + `lexer_*.ts`) - hand-written single-pass +lexers emitting flat token events (`Int32Array`) rendered to HTML in one +forward pass. `stylize` routes through it for ported languages (json, ts/js, +css, bash, html/markup + xml). Token types intern into a `TokenTypeRegistry` +(`token_types_global` by default, injectable via `SyntaxStylerOptions`). -**tokenize_syntax()** - Core tokenization function that processes text through -grammar patterns and returns a token stream. +**Regex engine** (`tokenize_syntax.ts` + `grammar_*.ts`) - PrismJS-inherited +multi-pass tokenization; still serves `tokenize()`, the unported languages +(svelte, md), and embedded regions inside unported grammars. + +**SyntaxStyler** - The main class for tokenization and HTML generation, +fronting both engines. + +**syntax_styler_global** - Pre-configured instance with all built-in +languages registered. Import this for typical usage. ### Token structure @@ -117,6 +131,19 @@ Generated HTML uses classes like `.token_keyword`, `.token_string`, styled by ### Language definitions +Lexer engine (preferred by `stylize` when registered): + +- `lexer_json.ts` - JSON (with comments) +- `lexer_ts.ts` - TypeScript; also registers the `js`/`javascript` aliases + (TS is a syntactic superset — there is no separate JS lexer) +- `lexer_css.ts` - CSS (including native nesting) +- `lexer_bash.ts` - Bash/shell +- `lexer_markup.ts` - HTML (`markup`/`html`/`mathml`/`svg`: rawtext + script/style/textarea/title, `style=`/`on*=` attribute embedding) and XML + (`xml`/`ssml`/`atom`/`rss`: plain tag scanning), one shared scanner + +Regex grammars (unported languages + `tokenize()` until cutover): + - `grammar_clike.ts` - base for C-like languages - `grammar_js.ts` - JavaScript - `grammar_ts.ts` - TypeScript (extends JS) @@ -199,8 +226,12 @@ in-tree baseline comparison the internal benchmark performs via runs the baseline compare on every invocation (free; just reads `benchmark/baseline.json`); only `--save` mutates it. -Internal benchmark tests all sample files at normal and 100x sizes. The vs -comparison tests against Prism and Shiki (JS and Oniguruma engines). +Internal benchmark tests all sample files at normal and 100x sizes, plus a +`pathological:` group of generated adversarial inputs (`src/test/pathological.ts`, +32KB each, shorter per-case budget) that tracks the lexer engine's worst-case +constants; the same generators back the CI linearity suite +(`src/test/lexer.pathological.test.ts`). The vs comparison tests against +Prism and Shiki (JS and Oniguruma engines). ### Updating committed result snapshots diff --git a/benchmark/benchmarks.ts b/benchmark/benchmarks.ts index ee69279a..1433a8af 100644 --- a/benchmark/benchmarks.ts +++ b/benchmark/benchmarks.ts @@ -11,12 +11,18 @@ import {format_file} from '@fuzdev/gro/format_file.ts'; import {samples as all_samples} from '../src/routes/samples/all.ts'; import {syntax_styler_global} from '../src/lib/syntax_styler_global.ts'; +import {PATHOLOGICAL_CASES} from '../src/test/pathological.ts'; /* eslint-disable no-console */ const BENCHMARK_TIME = 10000; const WARMUP_ITERATIONS = 50; const LARGE_CONTENT_MULTIPLIER = 100; +// pathological cases are regression tripwires, not headline numbers — a +// shorter budget keeps the full run's added cost modest while still giving +// the baseline compare thousands of iterations per case +const PATHOLOGICAL_BENCHMARK_TIME = 3000; +const PATHOLOGICAL_SIZE = 32768; const BASELINE_PATH = 'benchmark'; const BASELINE_FILE = `${BASELINE_PATH}/baseline.json`; @@ -60,7 +66,27 @@ export const run_benchmark = async (filter?: string): Promise { + syntax_styler_global.stylize(content, c.lang); + }); + pathological_count++; + } + if (pathological_count > 0) { + return results.concat(await pathological_bench.run()); + } + + return results; }; // `baseline:` and `large:` measure different workload sizes (1x vs 100x @@ -71,6 +97,7 @@ export const run_benchmark = async (filter?: string): Promise = [ {name: 'Baseline (1x content)', filter: (r) => r.name.startsWith('baseline:')}, {name: 'Large (100x content)', filter: (r) => r.name.startsWith('large:')}, + {name: 'Pathological (generated, 32KB)', filter: (r) => r.name.startsWith('pathological:')}, ]; // Heading kept inside the auto-managed region so the writer round-trips the diff --git a/src/lib/lexer.ts b/src/lib/lexer.ts index 8cb0613b..04e1f7bd 100644 --- a/src/lib/lexer.ts +++ b/src/lib/lexer.ts @@ -17,9 +17,9 @@ */ /** - * Interned metadata for a token type, shared by all lexers and stylers. + * Interned metadata for a token type. */ -export interface SyntaxTypeInfo { +export interface TokenTypeInfo { id: number; name: string; aliases: Array; @@ -33,35 +33,66 @@ export interface SyntaxTypeInfo { open_tag: string; } -const type_infos: Array = [ - // id 0 is reserved — it's the close-event tag - {id: 0, name: '', aliases: [], classes: '', open_tag: ''}, -]; -const type_ids: Map = new Map(); +/** + * An id space of interned token types with precomputed CSS classes and HTML + * open tags. Ids are only meaningful against the registry that interned them — + * a lexed event stream resolves back through the registry stamped on its + * `LexedSyntax`. + */ +export class TokenTypeRegistry { + /** + * Interned infos indexed by id. Hot loops hoist and index this directly; + * grow it only via `intern`. + */ + readonly infos: Array = [ + // id 0 is reserved — it's the close-event tag + {id: 0, name: '', aliases: [], classes: '', open_tag: ''}, + ]; + #ids: Map = new Map(); + + /** + * Interns a token type by name (+ optional aliases) and returns its id. + * Repeated calls with the same name and aliases return the same id. The CSS + * class list and HTML open tag are precomputed here so emitters never build + * class strings at runtime. + */ + intern(name: string, alias?: string | Array): number { + const aliases = alias === undefined ? [] : Array.isArray(alias) ? alias : [alias]; + const key = name + '\x1F' + aliases.join('\x1F'); + const existing = this.#ids.get(key); + if (existing !== undefined) return existing; + let classes = 'token_' + name; + for (const a of aliases) classes += ' token_' + a; + const id = this.infos.length; + this.infos.push({id, name, aliases, classes, open_tag: ''}); + this.#ids.set(key, id); + return id; + } + + /** + * Looks up the interned info for a token type id. + */ + info(id: number): TokenTypeInfo { + return this.infos[id]!; + } +} /** - * Interns a token type by name (+ optional aliases) into the global id space - * and returns its id. Repeated calls with the same name and aliases return the - * same id. The CSS class list and HTML open tag are precomputed here so - * emitters never build class strings at runtime. + * The shared default registry — the single id space used by the built-in + * lexers' module-load `token_type` constants and by any `SyntaxStyler` not + * given its own registry. The type vocabulary is global by design, mirroring + * the global `.token_*` CSS namespace; per-registry isolation exists for + * fully-custom stylers and tests, whose lexers must intern into the same + * registry they're rendered against. */ -export const token_type = (name: string, alias?: string | Array): number => { - const aliases = alias === undefined ? [] : Array.isArray(alias) ? alias : [alias]; - const key = name + '\x1F' + aliases.join('\x1F'); - const existing = type_ids.get(key); - if (existing !== undefined) return existing; - let classes = 'token_' + name; - for (const a of aliases) classes += ' token_' + a; - const id = type_infos.length; - type_infos.push({id, name, aliases, classes, open_tag: ''}); - type_ids.set(key, id); - return id; -}; +export const token_types_global: TokenTypeRegistry = new TokenTypeRegistry(); /** - * Looks up the interned info for a token type id. + * Interns a token type into `token_types_global` — the zero-config authoring + * path used by the built-in lexers' module-load type constants. */ -export const token_type_info = (id: number): SyntaxTypeInfo => type_infos[id]!; +export const token_type = (name: string, alias?: string | Array): number => + token_types_global.intern(name, alias); /** * A lexer-based language registration — the replacement for regex grammars. @@ -89,6 +120,12 @@ export interface LexedSyntax { text: string; events: Int32Array; events_len: number; + /** + * The registry that interned the type ids in `events` — consumers resolve + * ids through it, so a stream can never be rendered against the wrong + * id space. + */ + types: TokenTypeRegistry; } /** @@ -205,11 +242,14 @@ export class Lexer { * @param text - the source text * @param lang - the language to lex with * @param langs - registry used to resolve embedded languages by id + * @param types - token-type registry stamped on the result; must be the one + * `lang` (and any embedded language) interned its type ids into */ export const lex_syntax = ( text: string, lang: SyntaxLang, langs?: Map, + types: TokenTypeRegistry = token_types_global, ): LexedSyntax => { // capacity heuristic: dense token streams run ~1 int per source char const lexer = new Lexer(text.length); @@ -218,7 +258,7 @@ export const lex_syntax = ( lexer.end = text.length; lexer.langs = langs ?? null; lang.lex(lexer); - return {text, events: lexer.events, events_len: lexer.events_len}; + return {text, events: lexer.events, events_len: lexer.events_len, types}; }; /** @@ -251,6 +291,7 @@ const escape_html_slice = (text: string, from: number, to: number): string => { */ export const render_syntax_html = (lexed: LexedSyntax): string => { const {text, events, events_len} = lexed; + const {infos} = lexed.types; let out = ''; let pos = 0; let i = 0; @@ -260,13 +301,13 @@ export const render_syntax_html = (lexed: LexedSyntax): string => { const start = events[i + 1]!; const end = events[i + 2]!; if (start > pos) out += escape_html_slice(text, pos, start); - out += type_infos[tag]!.open_tag + escape_html_slice(text, start, end) + ''; + out += infos[tag]!.open_tag + escape_html_slice(text, start, end) + ''; pos = end; i += 3; } else if (tag < 0) { const start = events[i + 1]!; if (start > pos) out += escape_html_slice(text, pos, start); - out += type_infos[-tag]!.open_tag; + out += infos[-tag]!.open_tag; pos = start; i += 2; } else { @@ -297,16 +338,17 @@ export interface SyntaxEventToken { */ export const syntax_events_to_tokens = (lexed: LexedSyntax): Array => { const {events, events_len} = lexed; + const {infos} = lexed.types; const tokens: Array = []; const stack: Array = []; let i = 0; while (i < events_len) { const tag = events[i]!; if (tag > 0) { - tokens.push({type: type_infos[tag]!.name, start: events[i + 1]!, end: events[i + 2]!}); + tokens.push({type: infos[tag]!.name, start: events[i + 1]!, end: events[i + 2]!}); i += 3; } else if (tag < 0) { - const t: SyntaxEventToken = {type: type_infos[-tag]!.name, start: events[i + 1]!, end: -1}; + const t: SyntaxEventToken = {type: infos[-tag]!.name, start: events[i + 1]!, end: -1}; tokens.push(t); stack.push(t); i += 2; @@ -326,6 +368,7 @@ export const syntax_events_to_tokens = (lexed: LexedSyntax): Array => { const {text, events, events_len} = lexed; + const {infos} = lexed.types; const issues: Array = []; const stack: Array = []; let cursor = 0; @@ -339,7 +382,7 @@ export const validate_syntax_events = (lexed: LexedSyntax): Array => { } const start = events[i + 1]!; const end = events[i + 2]!; - if (tag >= type_infos.length) issues.push(`unknown type id ${tag} at ${i}`); + if (tag >= infos.length) issues.push(`unknown type id ${tag} at ${i}`); if (start < cursor) issues.push(`leaf start ${start} overlaps cursor ${cursor} at ${i}`); if (end <= start) issues.push(`empty or inverted leaf [${start}, ${end}) at ${i}`); if (end > text.length) issues.push(`leaf end ${end} out of bounds at ${i}`); @@ -351,7 +394,7 @@ export const validate_syntax_events = (lexed: LexedSyntax): Array => { break; } const start = events[i + 1]!; - if (-tag >= type_infos.length) issues.push(`unknown type id ${-tag} at ${i}`); + if (-tag >= infos.length) issues.push(`unknown type id ${-tag} at ${i}`); if (start < cursor) issues.push(`open start ${start} overlaps cursor ${cursor} at ${i}`); if (start > text.length) issues.push(`open start ${start} out of bounds at ${i}`); stack.push(start); diff --git a/src/lib/lexer_markup.ts b/src/lib/lexer_markup.ts new file mode 100644 index 00000000..197492ff --- /dev/null +++ b/src/lib/lexer_markup.ts @@ -0,0 +1,517 @@ +import {is_digit, is_space, skip_space, token_type, type Lexer, type SyntaxLang} from './lexer.ts'; + +/** + * Hand-written HTML/XML lexer. + * + * Token vocabulary matches the retired regex grammar: flat `comment`, + * `processing_instruction`, `doctype`, `cdata`, and `entity` + * (alias `named_entity` for the `&`-style form); a `tag` container holding + * a nested `tag` (the `` (the old pattern refused comments + * containing a second `', i + 4); + const comment_end = close === -1 || close + 3 > end ? end : close + 3; + l.leaf(T_COMMENT, i, comment_end); + return comment_end; + } + if (i + 9 <= end && matches_ci(text, i + 2, 'doctype')) { + let j = i + 9; + while (j < end) { + const c = text.charCodeAt(j); + if (c === 62) break; // > + if (c === 91) { + // [ internal subset — skip to the matching ] + const close_bracket = text.indexOf(']', j + 1); + j = close_bracket === -1 || close_bracket >= end ? end : close_bracket + 1; + continue; + } + j++; + } + const doctype_end = j < end ? j + 1 : end; + l.leaf(T_DOCTYPE, i, doctype_end); + return doctype_end; + } + if ( + i + 9 <= end && + text.charCodeAt(i + 2) === 91 && + matches_ci(text, i + 3, 'cdata') && + text.charCodeAt(i + 8) === 91 + ) { + const close = text.indexOf(']]>', i + 9); + const cdata_end = close === -1 || close + 3 > end ? end : close + 3; + l.leaf(T_CDATA, i, cdata_end); + return cdata_end; + } + } else if (c1 === 63) { + // ', i + 2); + const pi_end = close === -1 || close + 2 > end ? end : close + 2; + l.leaf(T_PROCESSING_INSTRUCTION, i, pi_end); + return pi_end; + } + const after_tag = lex_markup_tag(l, i, end, html_mode); + // rawtext elements — html mode, opening tags only; a self-closing slash is + // ignored (per HTML parsing, `', ['script', 'lang_ts', 'keyword']), [ + ['script', 'let x = 1;'], + ['lang_ts', 'let x = 1;'], + ['keyword', 'let'], + ]); + }); + + test('style embeds css', () => { + assert.deepEqual(picked('', ['style', 'lang_css', 'property']), [ + ['style', 'a{color:red}'], + ['lang_css', 'a{color:red}'], + ['property', 'color'], + ]); + }); + + test('textarea keeps expressions live', () => { + assert.deepEqual(picked('', ['svelte_expression']), [ + ['svelte_expression', '{value}'], + ]); + }); + + test('comments swallow expressions', () => { + assert.deepEqual(tokens_of(''), [['comment', '']]); + }); +}); + +describe('lexer_svelte sample', () => { + const content = readFileSync('src/test/fixtures/samples/sample_complex.svelte', 'utf8'); + + test('lexes the sample with valid invariants', () => { + assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(content, 'svelte')), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 11) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'svelte'); + assert.deepEqual(validate_syntax_events(lexed), [], `prefix of length ${len}`); + } + }); + + test('lexing is deterministic', () => { + const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'svelte')); + const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'svelte')); + assert.deepEqual(a, b); + }); +}); diff --git a/src/test/pathological.ts b/src/test/pathological.ts index 1a348964..eedc5e04 100644 --- a/src/test/pathological.ts +++ b/src/test/pathological.ts @@ -112,6 +112,18 @@ export const PATHOLOGICAL_CASES: Array = [ lang: 'html', generate: (size) => repeat_to_size('&x', size), }, + { + // expression-dense template text — balancer + ts embed per `{…}` + name: 'svelte_expression_dense', + lang: 'svelte', + generate: (size) => repeat_to_size('{x}', size), + }, + { + // expression-valued attributes — tag scanner + expression interplay + name: 'svelte_attr_expr_dense', + lang: 'svelte', + generate: (size) => repeat_to_size('', size), + }, { // nested rules NESTING_DEPTH deep — the native-nesting state stack name: 'css_deep_nesting', From ec30081b3aced32ffd93277ff1e80dc421c3a080 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 13:04:29 -0400 Subject: [PATCH 06/25] wip --- CLAUDE.md | 14 +- src/lib/lexer_markup.ts | 18 +- src/lib/lexer_md.ts | 414 ++++++++++++++++++ src/lib/syntax_styler_global.ts | 2 + .../fixtures/generated/md/md_complex.html | 74 ++-- src/test/fixtures/generated/md/md_complex.txt | 153 +++---- src/test/lexer_md.test.ts | 222 ++++++++++ src/test/pathological.ts | 12 + 8 files changed, 781 insertions(+), 128 deletions(-) create mode 100644 src/lib/lexer_md.ts create mode 100644 src/test/lexer_md.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index c55b6ef4..045394f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -68,7 +68,7 @@ src/ │ ├── syntax_styler.ts # SyntaxStyler class, hook system │ ├── syntax_styler_global.ts # pre-configured global instance │ ├── lexer.ts # lexer-engine substrate: Lexer, TokenTypeRegistry, flat events, HTML render -│ ├── lexer_*.ts # hand-written lexers (json, ts, css, bash, markup, svelte) +│ ├── lexer_*.ts # hand-written lexers (json, ts, css, bash, markup, svelte, md) │ ├── tokenize_syntax.ts # tokenize_syntax() function (regex engine) │ ├── syntax_token.ts # SyntaxToken class, type definitions │ ├── grammar_*.ts # regex language definitions (8 files) @@ -103,14 +103,14 @@ engine, language by language, on the `lexer-architecture` branch): **Lexer engine** (`lexer.ts` + `lexer_*.ts`) - hand-written single-pass lexers emitting flat token events (`Int32Array`) rendered to HTML in one -forward pass. `stylize` routes through it for ported languages (json, ts/js, -css, bash, html/markup + xml, svelte). Token types intern into a +forward pass. All 8 built-in languages are ported — `stylize` routes every +registered language through it. Token types intern into a `TokenTypeRegistry` (`token_types_global` by default, injectable via `SyntaxStylerOptions`). **Regex engine** (`tokenize_syntax.ts` + `grammar_*.ts`) - PrismJS-inherited -multi-pass tokenization; still serves `tokenize()`, the one unported language -(md), and embedded regions inside unported grammars. +multi-pass tokenization; now serves only the legacy `tokenize()` API and +explicit-grammar `stylize` calls, until its deletion completes the rewrite. **SyntaxStyler** - The main class for tokenization and HTML generation, fronting both engines. @@ -148,6 +148,10 @@ Lexer engine (preferred by `stylize` when registered): - `lexer_svelte.ts` - Svelte: the markup scanner in svelte mode (script→ts, no special attrs) plus the `{…}` expression lexer (blocks, each/await splits, at-directives, directive modifiers) +- `lexer_md.ts` - Markdown: line-oriented block scan (fences with exact-word + info matching and any-length closers, headings, blockquotes, lists, hr) + with a per-block inline scan (emphasis, inline code, links, entities, raw + markup via the markup scanner); fences embed their languages Regex grammars (unported languages + `tokenize()` until cutover): diff --git a/src/lib/lexer_markup.ts b/src/lib/lexer_markup.ts index 9a79cca6..f359be34 100644 --- a/src/lib/lexer_markup.ts +++ b/src/lib/lexer_markup.ts @@ -98,7 +98,11 @@ export interface MarkupLexMode { lex_expression: ((l: Lexer, from: number, end: number, full: boolean) => number) | null; } -const MODE_HTML: MarkupLexMode = { +/** + * The html dialect mode — also used by `lexer_md` for raw markup embedded in + * markdown text. + */ +export const MARKUP_MODE_HTML: MarkupLexMode = { script_embed: 'js', script_container: T_LANG_JS, rcdata: true, @@ -137,7 +141,7 @@ const is_entity_hex = (c: number): boolean => * -1. Mirrors the old patterns: named `&[\da-z]{1,8};` (case-insensitive) and * numeric `&#x?[\da-f]{1,8};` (hex digits allowed in both numeric forms). */ -const scan_entity_end = (text: string, i: number, end: number): number => { +export const scan_entity_end = (text: string, i: number, end: number): number => { let j = i + 1; const numeric = j < end && text.charCodeAt(j) === 35; // # if (numeric) { @@ -526,8 +530,14 @@ const lex_markup_rawtext = ( /** * Lexes one `<…` construct at `i`, returning the next scan position. + * Exported for `lexer_md`, which dispatches raw markup out of markdown text. */ -const lex_markup_construct = (l: Lexer, i: number, end: number, mode: MarkupLexMode): number => { +export const lex_markup_construct = ( + l: Lexer, + i: number, + end: number, + mode: MarkupLexMode, +): number => { const {text} = l; const c1 = i + 1 < end ? text.charCodeAt(i + 1) : 0; if (c1 === 33) { @@ -611,7 +621,7 @@ export const lex_markup_window = (l: Lexer, mode: MarkupLexMode): void => { }; const lex_markup_html = (l: Lexer): void => { - lex_markup_window(l, MODE_HTML); + lex_markup_window(l, MARKUP_MODE_HTML); }; const lex_markup_xml = (l: Lexer): void => { diff --git a/src/lib/lexer_md.ts b/src/lib/lexer_md.ts new file mode 100644 index 00000000..9758008f --- /dev/null +++ b/src/lib/lexer_md.ts @@ -0,0 +1,414 @@ +import {is_space, scan_to_line_end, token_type, type Lexer, type SyntaxLang} from './lexer.ts'; +import {lex_markup_construct, MARKUP_MODE_HTML, scan_entity_end} from './lexer_markup.ts'; + +/** + * Hand-written Markdown lexer — the structural rethink: a line-oriented block + * scan (fences, headings, blockquotes, lists, horizontal rules) with an + * inline scan per block (bold/italic/strikethrough, inline code, links, + * entities, and raw markup constructs dispatched through `lexer_markup`). + * + * Token vocabulary matches the retired regex grammar with one structurally + * forced collapse: the 27 per-language/per-size `fenced_code_*` types become + * a single `fenced_code` container (`code_fence` delimiter lines with the + * `punctuation` alias + a `lang_*` container embedding the fence language). + * Everything else keeps its name: `heading`/`blockquote`/`list` containers + * with aliased `punctuation`, `link` > `link_text_wrapper`/`url_wrapper`, + * `inline_code` (alias `code`) with `content`, `bold`/`italic`/ + * `strikethrough`, and markup's tag/comment/entity vocabulary for raw HTML. + * + * Fidelity fixes over the regex grammar (per the reviewed-diff policy): + * fence info words match exactly (` ```json ` embeds json — the old js + * alternation prefix-matched it as javascript) and closing fences accept any + * length ≥ the opener; interior lines starting with backticks are fence + * *content*, not stray `code_fence` tokens; heading/blockquote/list + * interiors get the inline scan (the old type-major engine left them plain); + * list containers span their line instead of swallowing preceding newlines; + * horizontal rules (`---`/`***`/`___`) get an `hr` token (alias + * `punctuation`). Emphasis delimiters keep the old matching rules + * (non-space-adjacent interiors free of the delimiter, word boundaries for + * `_` forms) but are line-bounded — the old engine let them span paragraphs. + * + * Resilience: an unterminated fence extends to the end of the window (the + * old engine re-lexed its interior as markdown). + */ + +const T_FENCED_CODE = token_type('fenced_code'); +const T_CODE_FENCE = token_type('code_fence', 'punctuation'); +const T_LANG_TS = token_type('lang_ts'); +const T_LANG_JS = token_type('lang_js'); +const T_LANG_CSS = token_type('lang_css'); +const T_LANG_MARKUP = token_type('lang_markup'); +const T_LANG_JSON = token_type('lang_json'); +const T_LANG_SVELTE = token_type('lang_svelte'); +const T_LANG_BASH = token_type('lang_bash'); +const T_LANG_MD = token_type('lang_md'); +const T_HEADING = token_type('heading'); +const T_HEADING_PUNCTUATION = token_type('punctuation', 'heading_punctuation'); +const T_BLOCKQUOTE = token_type('blockquote'); +const T_BLOCKQUOTE_PUNCTUATION = token_type('punctuation', 'blockquote_punctuation'); +const T_LIST = token_type('list'); +const T_PUNCTUATION = token_type('punctuation'); +const T_HR = token_type('hr', 'punctuation'); +const T_LINK = token_type('link'); +const T_LINK_TEXT_WRAPPER = token_type('link_text_wrapper'); +const T_LINK_TEXT = token_type('link_text'); +const T_LINK_PUNCTUATION = token_type('punctuation', 'link_punctuation'); +const T_URL_WRAPPER = token_type('url_wrapper'); +const T_URL = token_type('url'); +const T_INLINE_CODE = token_type('inline_code', 'code'); +const T_CODE_PUNCTUATION = token_type('punctuation', 'code_punctuation'); +const T_CONTENT = token_type('content'); +const T_BOLD = token_type('bold'); +const T_ITALIC = token_type('italic'); +const T_STRIKETHROUGH = token_type('strikethrough'); +const T_ENTITY = token_type('entity'); +const T_NAMED_ENTITY = token_type('entity', 'named_entity'); + +/** + * Fence info words → embedded language. Exact-word matching (the old + * alternations prefix-matched, misclassifying ` ```json ` as javascript). + */ +interface MdFenceLang { + id: string; + container: number; +} + +const FENCE_LANGS: Map = new Map(); +const add_fence_lang = (words: string, id: string, container: number): void => { + for (const word of words.split(' ')) FENCE_LANGS.set(word, {id, container}); +}; +add_fence_lang('ts typescript', 'ts', T_LANG_TS); +add_fence_lang('js javascript', 'js', T_LANG_JS); +add_fence_lang('css', 'css', T_LANG_CSS); +add_fence_lang('html markup', 'markup', T_LANG_MARKUP); +add_fence_lang('json', 'json', T_LANG_JSON); +add_fence_lang('svelte', 'svelte', T_LANG_SVELTE); +add_fence_lang('bash sh shell', 'bash', T_LANG_BASH); +add_fence_lang('md markdown', 'md', T_LANG_MD); + +// line-internal whitespace — `is_space` from the toolkit includes newlines +const is_line_space = (c: number): boolean => c === 32 || c === 9; + +// a `\w` word char, for the `_` emphasis word-boundary rule +const is_word = (c: number): boolean => + (c >= 48 && c <= 57) || ((c | 0x20) >= 97 && (c | 0x20) <= 122) || c === 95; + +const next_line_start = (text: string, from: number, end: number): number => { + const nl = text.indexOf('\n', from); + return nl === -1 || nl >= end ? end : nl + 1; +}; + +// whether [from, to) contains any non-space char +const has_inline_content = (text: string, from: number, to: number): boolean => { + for (let i = from; i < to; i++) { + if (!is_line_space(text.charCodeAt(i))) return true; + } + return false; +}; + +/** + * Lexes an emphasis form at the delimiter `code` at `i` — the double form + * (`**`/`__`/`~~`) into `double_type`, falling back to the single form + * (`*`/`_`) into `italic` when allowed. Old matching rules: the interior is + * free of the delimiter char with non-space first/last chars; `_` forms + * require word boundaries outside. Returns the next scan position. + */ +const lex_md_emphasis = ( + l: Lexer, + i: number, + line_end: number, + code: number, + ds: string, + double_type: number, + word_boundary: boolean, + allow_single: boolean, +): number => { + const {text} = l; + if (word_boundary && is_word(text.charCodeAt(i - 1))) return i + 1; + if (text.charCodeAt(i + 1) === code) { + // double form — interior is [i+2, k), closer [k, k+2) + const k = text.indexOf(ds, i + 2); + if ( + k !== -1 && + k + 1 < line_end && + k > i + 2 && + text.charCodeAt(k + 1) === code && + !is_space(text.charCodeAt(i + 2)) && + !is_space(text.charCodeAt(k - 1)) && + (!word_boundary || !is_word(text.charCodeAt(k + 2))) + ) { + l.open(double_type, i); + l.leaf(T_PUNCTUATION, i, i + 2); + l.leaf(T_PUNCTUATION, k, k + 2); + l.close(k + 2); + return k + 2; + } + if (!allow_single) return i + 1; + } + if (!allow_single) return i + 1; + // single form — interior is [i+1, k) + const k = text.indexOf(ds, i + 1); + if ( + k !== -1 && + k < line_end && + k > i + 1 && + !is_space(text.charCodeAt(i + 1)) && + !is_space(text.charCodeAt(k - 1)) && + (!word_boundary || !is_word(text.charCodeAt(k + 1))) + ) { + l.open(T_ITALIC, i); + l.leaf(T_PUNCTUATION, i, i + 1); + l.leaf(T_PUNCTUATION, k, k + 1); + l.close(k + 1); + return k + 1; + } + return i + 1; +}; + +/** + * Lexes a `[text](url)` link at the `[` at `i`, returning the next scan + * position (`i + 1` when it isn't a link). Old constraints: text interior + * free of `[`/`]`, url interior free of `)`, both line-bounded and non-empty, + * `(` immediately after `]`. + */ +const lex_md_link = (l: Lexer, i: number, line_end: number): number => { + const {text} = l; + const rb = text.indexOf(']', i + 1); + if (rb === -1 || rb >= line_end || rb === i + 1) return i + 1; + const lb2 = text.indexOf('[', i + 1); + if (lb2 !== -1 && lb2 < rb) return i + 1; + if (text.charCodeAt(rb + 1) !== 40) return i + 1; + const cp = text.indexOf(')', rb + 2); + if (cp === -1 || cp >= line_end || cp === rb + 2) return i + 1; + l.open(T_LINK, i); + l.open(T_LINK_TEXT_WRAPPER, i); + l.leaf(T_LINK_PUNCTUATION, i, i + 1); + l.leaf(T_LINK_TEXT, i + 1, rb); + l.leaf(T_LINK_PUNCTUATION, rb, rb + 1); + l.close(rb + 1); + l.open(T_URL_WRAPPER, rb + 1); + l.leaf(T_LINK_PUNCTUATION, rb + 1, rb + 2); + l.leaf(T_URL, rb + 2, cp); + l.leaf(T_LINK_PUNCTUATION, cp, cp + 1); + l.close(cp + 1); + l.close(cp + 1); + return cp + 1; +}; + +/** + * The inline scan — bold/italic/strikethrough, inline code, links, entities, + * and raw markup constructs, over `[from, to)`. Markup constructs are + * allowed to extend to `construct_end` (the window end in paragraph context, + * so multi-line tags/comments/script regions work; the line end inside block + * containers, so container nesting stays valid). Returns the final position, + * which exceeds `to` only when a paragraph construct spanned lines. + */ +const lex_md_inline = (l: Lexer, from: number, to: number, construct_end: number): number => { + const {text} = l; + let i = from; + let line_end = to; + while (i < line_end) { + const c = text.charCodeAt(i); + if (c === 60) { + // < — raw markup construct + i = lex_markup_construct(l, i, construct_end, MARKUP_MODE_HTML); + if (i > line_end) line_end = scan_to_line_end(text, i, construct_end); + continue; + } + if (c === 38) { + // & — entity + const entity_end = scan_entity_end(text, i, line_end); + if (entity_end === -1) { + i++; + } else { + l.leaf(text.charCodeAt(i + 1) === 35 ? T_ENTITY : T_NAMED_ENTITY, i, entity_end); + i = entity_end; + } + continue; + } + if (c === 96) { + // ` — inline code, single backticks, line-bounded, non-empty + const close = text.indexOf('`', i + 1); + if (close !== -1 && close < line_end && close > i + 1) { + l.open(T_INLINE_CODE, i); + l.leaf(T_CODE_PUNCTUATION, i, i + 1); + l.leaf(T_CONTENT, i + 1, close); + l.leaf(T_CODE_PUNCTUATION, close, close + 1); + l.close(close + 1); + i = close + 1; + } else { + i++; + } + continue; + } + if (c === 42) { + // * + i = lex_md_emphasis(l, i, line_end, 42, '*', T_BOLD, false, true); + continue; + } + if (c === 95) { + // _ + i = lex_md_emphasis(l, i, line_end, 95, '_', T_BOLD, true, true); + continue; + } + if (c === 126) { + // ~ + i = lex_md_emphasis(l, i, line_end, 126, '~', T_STRIKETHROUGH, false, false); + continue; + } + if (c === 91) { + // [ + i = lex_md_link(l, i, line_end); + continue; + } + i++; + } + return i; +}; + +/** + * Lexes a fenced code block from the opening backticks at `ls` (col 0, ≥3), + * returning the position after the block. The close fence is a col-0 line of + * ≥ the opening count of backticks and nothing else; an unterminated fence + * extends to the window end. Known info words embed their language inside a + * `lang_*` container; unknown/absent info leaves the content plain. + */ +const lex_md_fence = (l: Lexer, ls: number, le: number, next: number, end: number): number => { + const {text} = l; + let bt = ls; + while (bt < le && text.charCodeAt(bt) === 96) bt++; + const count = bt - ls; + // info word — immediately after the backticks (matching the old patterns), + // ending at whitespace or the line end + let w = bt; + while (w < le && !is_line_space(text.charCodeAt(w))) w++; + const lang = w > bt ? FENCE_LANGS.get(text.slice(bt, w)) : undefined; + + // find the close fence line + let close_ls = -1; + let close_le = end; + let search = next; + while (search < end) { + const sle = scan_to_line_end(text, search, end); + if (text.charCodeAt(search) === 96) { + let b2 = search; + while (b2 < sle && text.charCodeAt(b2) === 96) b2++; + if (b2 - search >= count && b2 === sle) { + close_ls = search; + close_le = sle; + break; + } + } + search = next_line_start(text, search, end); + } + + l.open(T_FENCED_CODE, ls); + l.leaf(T_CODE_FENCE, ls, le); + const content_start = le; // includes the newline, matching the old spans + const content_end = close_ls === -1 ? end : close_ls; + if (lang !== undefined && content_end > content_start) { + l.open(lang.container, content_start); + l.embed(lang.id, content_start, content_end); + l.close(content_end); + } + if (close_ls === -1) { + l.close(end); + return end; + } + l.leaf(T_CODE_FENCE, close_ls, close_le); + l.close(close_le); + return next_line_start(text, close_le, end); +}; + +/** + * Lexes one line starting at `ls`, returning the next scan position. + */ +const lex_md_line = (l: Lexer, ls: number, le: number, next: number, end: number): number => { + const {text} = l; + const c = text.charCodeAt(ls); + + // fenced code block — col 0, 3+ backticks + if (c === 96 && text.charCodeAt(ls + 1) === 96 && text.charCodeAt(ls + 2) === 96) { + return lex_md_fence(l, ls, le, next, end); + } + + // heading — 1-6 `#`s, then whitespace, then content + if (c === 35) { + let h = ls + 1; + while (h < le && h - ls < 6 && text.charCodeAt(h) === 35) h++; + if ( + h < le && + text.charCodeAt(h) !== 35 && + is_line_space(text.charCodeAt(h)) && + has_inline_content(text, h + 1, le) + ) { + l.open(T_HEADING, ls); + l.leaf(T_HEADING_PUNCTUATION, ls, h); + lex_md_inline(l, h, le, le); + l.close(le); + return next; + } + } + + // blockquote — col-0 `>` with content + if (c === 62 && has_inline_content(text, ls + 1, le)) { + l.open(T_BLOCKQUOTE, ls); + l.leaf(T_BLOCKQUOTE_PUNCTUATION, ls, ls + 1); + lex_md_inline(l, ls + 1, le, le); + l.close(le); + return next; + } + + // horizontal rule — a col-0 line of 3+ identical `-`/`*`/`_` + if (c === 45 || c === 42 || c === 95) { + let k = ls; + while (k < le && text.charCodeAt(k) === c) k++; + if (k === le && k - ls >= 3) { + l.leaf(T_HR, ls, le); + return next; + } + } + + // list item — optional indent, `-`/`*` marker, whitespace, content + { + let m = ls; + while (m < le && is_line_space(text.charCodeAt(m))) m++; + const mc = text.charCodeAt(m); + if ( + (mc === 45 || mc === 42) && + m + 1 < le && + is_line_space(text.charCodeAt(m + 1)) && + has_inline_content(text, m + 2, le) + ) { + l.open(T_LIST, ls); + l.leaf(T_PUNCTUATION, ls, m + 1); + lex_md_inline(l, m + 1, le, le); + l.close(le); + return next; + } + } + + // paragraph text — markup constructs may span lines + const after = lex_md_inline(l, ls, le, end); + return next_line_start(text, after > le ? after : le, end); +}; + +const lex_md = (l: Lexer): void => { + const {text, end} = l; + let i = l.pos; + while (i < end) { + const le = scan_to_line_end(text, i, end); + if (le === i) { + // blank line + i = next_line_start(text, i, end); + continue; + } + i = lex_md_line(l, i, le, next_line_start(text, i, end), end); + } + l.pos = end; +}; + +/** + * The Markdown language registration for the lexer engine. + */ +export const lexer_md: SyntaxLang = {id: 'md', aliases: ['markdown'], lex: lex_md}; diff --git a/src/lib/syntax_styler_global.ts b/src/lib/syntax_styler_global.ts index a68272a5..d05a0d02 100644 --- a/src/lib/syntax_styler_global.ts +++ b/src/lib/syntax_styler_global.ts @@ -5,6 +5,7 @@ import {lexer_css} from './lexer_css.ts'; import {lexer_bash} from './lexer_bash.ts'; import {lexer_markup, lexer_xml} from './lexer_markup.ts'; import {lexer_svelte} from './lexer_svelte.ts'; +import {lexer_md} from './lexer_md.ts'; import {add_grammar_markup} from './grammar_markup.ts'; import {add_grammar_css} from './grammar_css.ts'; import {add_grammar_clike} from './grammar_clike.ts'; @@ -47,3 +48,4 @@ syntax_styler_global.add_lexer_lang(lexer_bash); syntax_styler_global.add_lexer_lang(lexer_markup); syntax_styler_global.add_lexer_lang(lexer_xml); syntax_styler_global.add_lexer_lang(lexer_svelte); +syntax_styler_global.add_lexer_lang(lexer_md); diff --git a/src/test/fixtures/generated/md/md_complex.html b/src/test/fixtures/generated/md/md_complex.html index 4f20f861..a07d7fea 100644 --- a/src/test/fixtures/generated/md/md_complex.html +++ b/src/test/fixtures/generated/md/md_complex.html @@ -22,32 +22,32 @@ This is a [link to example](https://example.com) in text. -## Lists +## Lists -- list item 1 -- list item 2 -- list item 3 +- list item 1 +- list item 2 +- list item 3 -* alternative list item 1 -* alternative list item 2 - - indented list item 1 - - indented list item 2 - - Deeply nested item +* alternative list item 1 +* alternative list item 2 + - indented list item 1 + - indented list item 2 + - Deeply nested item - - after a newline + - after a newline ## Blockquotes > This is a blockquote. > It can span multiple lines. -> Another blockquote with **bold** and _italic_. +> Another blockquote with **bold** and _italic_. ## Fenced Code Blocks ### TypeScript -```ts +```ts function add(a: number, b: number): number { return a + b; } @@ -55,32 +55,32 @@ #### TypeScript with alias 'typescript' -```typescript -interface User { +```typescript +interface User { name: string; age: number; } -const user: User = {name: 'Alice', age: 30}; +const user: User = {name: 'Alice', age: 30}; ``` ### JS -```js -function add(a, b) { +```js +function add(a, b) { return a + b; } ``` #### JS with alias 'javascript' -```javascript -const fifteen = multiply(5, 3); +```javascript +const fifteen = multiply(5, 3); ``` ### CSS -```css +```css .container { display: flex; font-size: 14px; @@ -89,7 +89,7 @@ ### HTML -```html +```html <!doctype html> <div class="test"> <p>hello world!</p> @@ -102,7 +102,7 @@ #### HTML with alias 'markup' -```markup +```markup <ul> <li>a</li> <li>b</li> @@ -111,22 +111,22 @@ ### JSON -```json +```json { - "key": "value", - "array": [1, 2, 3, true, false, null] + "key": "value", + "array": [1, 2, 3, true, false, null] } ``` ### Svelte -```svelte +```svelte <script lang="ts"> - let count: number = $state(0); - const increment = () => count++; + let count: number = $state(0); + const increment = () => count++; </script> -<button type="button" onclick={increment}> +<button type="button" onclick={increment}> Count: {count} </button> @@ -139,8 +139,8 @@ ### Bash -```bash -if [[ $name =~ ^[a-z]+$ ]]; then +```bash +if [[ $name =~ ^[a-z]+$ ]]; then echo "hello $(whoami)" fi ``` @@ -175,18 +175,18 @@ Empty fence above. -```unknownlang +```unknownlang Plain text for unknown language. const variable = 2; ``` Nested fences (3 to 5): -`````md +`````md with `five` fences -````md +````md with `four` fences -```html +```html <style> body { color: red; } </style> @@ -194,7 +194,7 @@ ending `four` ```` another `four` -````ts +````ts const a = '5'; ```` ending `five` @@ -212,7 +212,7 @@ <div> -```ts +```ts const example = 'embedded code'; ``` diff --git a/src/test/fixtures/generated/md/md_complex.txt b/src/test/fixtures/generated/md/md_complex.txt index 0239de59..4e9a2209 100644 --- a/src/test/fixtures/generated/md/md_complex.txt +++ b/src/test/fixtures/generated/md/md_complex.txt @@ -1,34 +1,31 @@ === STATS === Sample length: 2799 characters -Total tokens: 527 +Total tokens: 525 -Token types (60 unique): - punctuation: 225 +Token types (51 unique): + punctuation: 223 tag: 56 - code_fence: 35 + code_fence: 34 heading: 23 operator: 20 - keyword: 13 + fenced_code: 17 + keyword: 12 list: 9 + bold: 7 + italic: 7 inline_code: 7 content: 7 lang_ts: 7 builtin: 7 number: 7 attr_name: 7 - bold: 6 - italic: 6 + property: 6 + attr_value: 6 string: 5 - attr_value: 5 function: 4 - property: 4 blockquote: 3 - fenced_code_ts: 3 - fenced_code_js: 3 - lang_js: 3 lang_css: 3 selector: 3 - fenced_code_html: 3 lang_markup: 3 strikethrough: 2 link: 2 @@ -37,32 +34,26 @@ Token types (60 unique): url_wrapper: 2 url: 2 special_keyword: 2 - capitalized_identifier: 2 type_annotation: 2 :: 2 type: 2 - string_property: 2 + lang_js: 2 boolean: 2 svelte_expression: 2 style: 2 - fenced_code: 2 lang_md: 2 class_name: 1 - parameter: 1 - fenced_code_css: 1 + type_name: 1 doctype: 1 comment: 1 - fenced_code_svelte: 1 + lang_json: 1 + null: 1 lang_svelte: 1 script: 1 function_variable: 1 - fenced_code_bash: 1 lang_bash: 1 variable: 1 command_substitution: 1 - fenced_code_5tick_md: 1 - fenced_code_4tick_md: 1 - fenced_code_4tick_ts: 1 === TOKENS === 0-11 heading # Heading 1 @@ -109,24 +100,24 @@ Token types (60 unique): 344-345 punctuation ) 356-364 heading ## Lists 356-358 punctuation ## - 364-379 list \n\n- list item 1 - 364-367 punctuation \n\n- - 379-393 list \n- list item 2 - 379-381 punctuation \n- - 393-407 list \n- list item 3 - 393-395 punctuation \n- - 407-434 list \n\n* alternative list item 1 - 407-410 punctuation \n\n* - 434-460 list \n* alternative list item 2 - 434-436 punctuation \n* - 460-485 list \n - indented list item 1 - 460-464 punctuation \n - - 485-510 list \n - indented list item 2 - 485-489 punctuation \n - - 510-535 list \n - Deeply nested item - 510-516 punctuation \n - - 535-556 list \n\n - after a newline - 535-540 punctuation \n\n - + 366-379 list - list item 1 + 366-367 punctuation - + 380-393 list - list item 2 + 380-381 punctuation - + 394-407 list - list item 3 + 394-395 punctuation - + 409-434 list * alternative list item 1 + 409-410 punctuation * + 435-460 list * alternative list item 2 + 435-436 punctuation * + 461-485 list - indented list item 1 + 461-464 punctuation - + 486-510 list - indented list item 2 + 486-489 punctuation - + 511-535 list - Deeply nested item + 511-516 punctuation - + 537-556 list - after a newline + 537-540 punctuation - 558-572 heading ## Blockquotes 558-560 punctuation ## 574-597 blockquote > This is a blockquote. @@ -135,11 +126,17 @@ Token types (60 unique): 598-599 punctuation > 629-677 blockquote > Another blockquote with **bold** and _italic_. 629-630 punctuation > + 655-663 bold **bold** + 655-657 punctuation ** + 661-663 punctuation ** + 668-676 italic _italic_ + 668-669 punctuation _ + 675-676 punctuation _ 679-700 heading ## Fenced Code Blocks 679-681 punctuation ## 702-716 heading ### TypeScript 702-705 punctuation ### - 718-789 fenced_code_ts ```ts\nfunction add(a: number, b: number): number {\n\treturn a + b;\n}\n``` + 718-789 fenced_code ```ts\nfunction add(a: number, b: number): number {\n\treturn a + b;\n}\n``` 718-723 code_fence ```ts 723-786 lang_ts \nfunction add(a: number, b: number): number {\n\treturn a + b;\n}\n 724-732 keyword function @@ -161,12 +158,11 @@ Token types (60 unique): 786-789 code_fence ``` 791-830 heading #### TypeScript with alias 'typescript' 791-795 punctuation #### - 832-943 fenced_code_ts ```typescript\ninterface User {\n\tname: string;\n\tage: number;\n}\n\nconst user: User = {name: 'Alice', age: 30};\n``` + 832-943 fenced_code ```typescript\ninterface User {\n\tname: string;\n\tage: number;\n}\n\nconst user: User = {name: 'Alice', age: 30};\n``` 832-845 code_fence ```typescript 845-940 lang_ts \ninterface User {\n\tname: string;\n\tage: number;\n}\n\nconst user: User = {name: 'Alice', age: 30};\n 846-855 keyword interface 856-860 class_name User - 856-860 capitalized_identifier User 861-862 punctuation { 868-869 operator : 870-876 builtin string @@ -176,10 +172,10 @@ Token types (60 unique): 890-891 punctuation ; 892-893 punctuation } 895-900 keyword const - 905-912 type_annotation : User + 905-911 type_annotation : User 905-906 : : - 906-912 type User - 907-911 capitalized_identifier User + 907-911 type User + 907-911 type_name User 912-913 operator = 914-915 punctuation { 919-920 operator : @@ -187,18 +183,16 @@ Token types (60 unique): 928-929 punctuation , 933-934 operator : 935-937 number 30 - 937-938 punctuation } - 938-939 punctuation ; + 937-939 punctuation }; 940-943 code_fence ``` 945-951 heading ### JS 945-948 punctuation ### - 953-1000 fenced_code_js ```js\nfunction add(a, b) {\n\treturn a + b;\n}\n``` + 953-1000 fenced_code ```js\nfunction add(a, b) {\n\treturn a + b;\n}\n``` 953-958 code_fence ```js 958-997 lang_js \nfunction add(a, b) {\n\treturn a + b;\n}\n 959-967 keyword function 968-971 function add 971-972 punctuation ( - 972-976 parameter a, b 973-974 punctuation , 976-977 punctuation ) 978-979 punctuation { @@ -209,7 +203,7 @@ Token types (60 unique): 997-1000 code_fence ``` 1002-1033 heading #### JS with alias 'javascript' 1002-1006 punctuation #### -1035-1084 fenced_code_js ```javascript\nconst fifteen = multiply(5, 3);\n``` +1035-1084 fenced_code ```javascript\nconst fifteen = multiply(5, 3);\n``` 1035-1048 code_fence ```javascript 1048-1081 lang_js \nconst fifteen = multiply(5, 3);\n 1049-1054 keyword const @@ -219,12 +213,11 @@ Token types (60 unique): 1074-1075 number 5 1075-1076 punctuation , 1077-1078 number 3 -1078-1079 punctuation ) -1079-1080 punctuation ; +1078-1080 punctuation ); 1081-1084 code_fence ``` 1086-1093 heading ### CSS 1086-1089 punctuation ### -1095-1154 fenced_code_css ```css\n.container {\n\tdisplay: flex;\n\tfont-size: 14px;\n}\n``` +1095-1154 fenced_code ```css\n.container {\n\tdisplay: flex;\n\tfont-size: 14px;\n}\n``` 1095-1101 code_fence ```css 1101-1151 lang_css \n.container {\n\tdisplay: flex;\n\tfont-size: 14px;\n}\n 1102-1112 selector .container @@ -239,7 +232,7 @@ Token types (60 unique): 1151-1154 code_fence ``` 1156-1164 heading ### HTML 1156-1159 punctuation ### -1166-1308 fenced_code_html ```html\n\n
\n\t

hello world!

\n
\n\n\n\n\n``` +1166-1308 fenced_code ```html\n\n
\n\t

hello world!

\n
\n\n\n\n\n``` 1166-1173 code_fence ```html 1173-1305 lang_markup \n\n
\n\t

hello world!

\n
\n\n\n\n\n 1174-1189 doctype @@ -282,7 +275,7 @@ Token types (60 unique): 1305-1308 code_fence ``` 1310-1339 heading #### HTML with alias 'markup' 1310-1314 punctuation #### -1341-1389 fenced_code_html ```markup\n
    \n\t
  • a
    • \n\t
  • b
    • \n
\n``` +1341-1389 fenced_code ```markup\n
    \n\t
  • a
    • \n\t
  • b
    • \n
\n``` 1341-1350 code_fence ```markup 1350-1386 lang_markup \n
    \n\t
  • a
    • \n\t
  • b
    • \n
\n 1351-1355 tag
    @@ -312,15 +305,15 @@ Token types (60 unique): 1386-1389 code_fence ``` 1391-1399 heading ### JSON 1391-1394 punctuation ### -1401-1472 fenced_code_js ```json\n{\n\t"key": "value",\n\t"array": [1, 2, 3, true, false, null]\n}\n``` +1401-1472 fenced_code ```json\n{\n\t"key": "value",\n\t"array": [1, 2, 3, true, false, null]\n}\n``` 1401-1408 code_fence ```json -1408-1469 lang_js \n{\n\t"key": "value",\n\t"array": [1, 2, 3, true, false, null]\n}\n +1408-1469 lang_json \n{\n\t"key": "value",\n\t"array": [1, 2, 3, true, false, null]\n}\n 1409-1410 punctuation { -1412-1417 string_property "key" +1412-1417 property "key" 1417-1418 operator : 1419-1426 string "value" 1426-1427 punctuation , -1429-1436 string_property "array" +1429-1436 property "array" 1436-1437 operator : 1438-1439 punctuation [ 1439-1440 number 1 @@ -333,13 +326,13 @@ Token types (60 unique): 1452-1453 punctuation , 1454-1459 boolean false 1459-1460 punctuation , -1461-1465 keyword null +1461-1465 null null 1465-1466 punctuation ] 1467-1468 punctuation } 1469-1472 code_fence ``` 1474-1484 heading ### Svelte 1474-1477 punctuation ### -1486-1716 fenced_code_svelte ```svelte\n\n\n\n\n\n``` +1486-1716 fenced_code ```svelte\n\n\n\n\n\n``` 1486-1495 code_fence ```svelte 1495-1713 lang_svelte \n\n\n\n\n\n 1496-1514 tag ', ['script', 'lang_js', 'keyword']), [ + ['script', '\nlet x = 1;\n'], + ['lang_js', '\nlet x = 1;\n'], + ['keyword', 'let'], + ]); + }); + + test('markup constructs inside block containers clamp to the line', () => { + const text = '> quote { + const content = readFileSync('src/test/fixtures/samples/sample_complex.md', 'utf8'); + + test('lexes the sample with valid invariants', () => { + assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(content, 'md')), []); + }); + + test('every prefix lexes without throwing, with valid invariants', () => { + for (let len = 0; len <= content.length; len += 13) { + const prefix = content.slice(0, len); + const lexed = syntax_styler_global.lex(prefix, 'md'); + assert.deepEqual(validate_syntax_events(lexed), [], `prefix of length ${len}`); + } + }); + + test('lexing is deterministic', () => { + const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'md')); + const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'md')); + assert.deepEqual(a, b); + }); +}); diff --git a/src/test/pathological.ts b/src/test/pathological.ts index eedc5e04..c102d4ea 100644 --- a/src/test/pathological.ts +++ b/src/test/pathological.ts @@ -118,6 +118,18 @@ export const PATHOLOGICAL_CASES: Array = [ lang: 'svelte', generate: (size) => repeat_to_size('{x}', size), }, + { + // emphasis-delimiter-dense text — every `*`/`_` probes for a closer + name: 'md_emphasis_dense', + lang: 'md', + generate: (size) => repeat_to_size('*a _b ', size), + }, + { + // fence-dense markdown — block scanner + close-fence line scans + name: 'md_fence_dense', + lang: 'md', + generate: (size) => repeat_to_size('```ts\nlet x = 1;\n```\n', size), + }, { // expression-valued attributes — tag scanner + expression interplay name: 'svelte_attr_expr_dense', From e3bd49df67322715930b66105bdabe0b7f4980a6 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 13:38:43 -0400 Subject: [PATCH 07/25] wip --- src/lib/lexer_markup.ts | 7 ++++++- src/lib/lexer_md.ts | 39 +++++++++++++++++++++++++++++------ src/lib/lexer_ts.ts | 6 +++--- src/test/lexer_svelte.test.ts | 20 ++++++++++++++++++ src/test/pathological.ts | 14 +++++++++++++ 5 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/lib/lexer_markup.ts b/src/lib/lexer_markup.ts index f359be34..1c3af57a 100644 --- a/src/lib/lexer_markup.ts +++ b/src/lib/lexer_markup.ts @@ -261,9 +261,14 @@ const lex_markup_attr_value = ( const lex_markup_attr_name = (l: Lexer, from: number, to: number, mode: MarkupLexMode): void => { const {text} = l; const ns_end = scan_namespace_end(text, from, to); + // directive-modifier pipes come *after* any `ns:` prefix — a `|` inside the + // namespace span (malformed input like `a|b:c`) is part of the namespace, + // not a modifier separator, so start the pipe scan at `ns_end` to keep the + // emitted leaves monotonic (a pipe leaf before `ns_end` would overlap the + // namespace leaf and produce an invalid event stream) let pipe = -1; if (mode.lex_expression !== null) { - for (let k = from; k < to; k++) { + for (let k = ns_end === -1 ? from : ns_end; k < to; k++) { if (text.charCodeAt(k) === 124) { pipe = k; break; diff --git a/src/lib/lexer_md.ts b/src/lib/lexer_md.ts index 9758008f..6c677332 100644 --- a/src/lib/lexer_md.ts +++ b/src/lib/lexer_md.ts @@ -165,21 +165,46 @@ const lex_md_emphasis = ( return i + 1; }; +/** + * Monotonic next-occurrence caches for `lex_md_link`'s `]` and `)` searches. + * Because a `[` searches for a *different* char, a `[`-dense (or `[x](`-dense) + * line would otherwise pay a full forward `indexOf` per `[` while advancing + * only one char — O(n²). The caches advance forward through the inline scan + * (which visits `[` positions in increasing order), so total probe work is + * O(n). `Infinity` means the char has no further occurrence. + */ +interface MdLinkCache { + rbracket: number; + rparen: number; +} + +/** + * Returns the cached next occurrence of `ch` at or after `from`, re-probing + * with `indexOf` only when the cached position has fallen behind `from`. + */ +const md_probe = (text: string, cached: number, from: number, ch: string): number => { + if (cached >= from) return cached; + const found = text.indexOf(ch, from); + return found === -1 ? Infinity : found; +}; + /** * Lexes a `[text](url)` link at the `[` at `i`, returning the next scan * position (`i + 1` when it isn't a link). Old constraints: text interior * free of `[`/`]`, url interior free of `)`, both line-bounded and non-empty, * `(` immediately after `]`. */ -const lex_md_link = (l: Lexer, i: number, line_end: number): number => { +const lex_md_link = (l: Lexer, i: number, line_end: number, cache: MdLinkCache): number => { const {text} = l; - const rb = text.indexOf(']', i + 1); - if (rb === -1 || rb >= line_end || rb === i + 1) return i + 1; + cache.rbracket = md_probe(text, cache.rbracket, i + 1, ']'); + const rb = cache.rbracket; + if (rb >= line_end || rb === i + 1) return i + 1; const lb2 = text.indexOf('[', i + 1); if (lb2 !== -1 && lb2 < rb) return i + 1; if (text.charCodeAt(rb + 1) !== 40) return i + 1; - const cp = text.indexOf(')', rb + 2); - if (cp === -1 || cp >= line_end || cp === rb + 2) return i + 1; + cache.rparen = md_probe(text, cache.rparen, rb + 2, ')'); + const cp = cache.rparen; + if (cp >= line_end || cp === rb + 2) return i + 1; l.open(T_LINK, i); l.open(T_LINK_TEXT_WRAPPER, i); l.leaf(T_LINK_PUNCTUATION, i, i + 1); @@ -205,6 +230,8 @@ const lex_md_link = (l: Lexer, i: number, line_end: number): number => { */ const lex_md_inline = (l: Lexer, from: number, to: number, construct_end: number): number => { const {text} = l; + // per-call caches so a `[`-dense line stays linear (see `md_probe`) + const link_cache: MdLinkCache = {rbracket: -1, rparen: -1}; let i = from; let line_end = to; while (i < line_end) { @@ -258,7 +285,7 @@ const lex_md_inline = (l: Lexer, from: number, to: number, construct_end: number } if (c === 91) { // [ - i = lex_md_link(l, i, line_end); + i = lex_md_link(l, i, line_end, link_cache); continue; } i++; diff --git a/src/lib/lexer_ts.ts b/src/lib/lexer_ts.ts index fbfb7976..84bae44c 100644 --- a/src/lib/lexer_ts.ts +++ b/src/lib/lexer_ts.ts @@ -493,7 +493,7 @@ const lex_ts_template = (l: Lexer, i: number, to: number, cache: TsScanCache): n j++; closed = true; break; - } else if (c === 36 && text.charCodeAt(j + 1) === 123) { + } else if (c === 36 && j + 1 < to && text.charCodeAt(j + 1) === 123) { l.leaf(T_STRING, chunk_start, j); const close = scan_balanced_braces(text, j + 1, to); const inner_end = close === -1 ? to : close; @@ -548,7 +548,7 @@ const lex_ts_window = ( } // identifiers (including `#private`) - if (is_ident_start(c) || (c === 35 && is_ident_start(text.charCodeAt(i + 1)))) { + if (is_ident_start(c) || (c === 35 && i + 1 < to && is_ident_start(text.charCodeAt(i + 1)))) { const start = i; const ident_end = scan_ident(text, c === 35 ? i + 1 : i, to); const was_class_ctx = class_ctx; @@ -827,7 +827,7 @@ const lex_ts_window = ( } // `@decorator` - if (c === 64 && is_ident_start(text.charCodeAt(i + 1))) { + if (c === 64 && i + 1 < to && is_ident_start(text.charCodeAt(i + 1))) { const name_end = scan_ident(text, i + 1, to); l.open(T_DECORATOR, i); l.leaf(T_AT, i, i + 1); diff --git a/src/test/lexer_svelte.test.ts b/src/test/lexer_svelte.test.ts index 485c6690..1c590f83 100644 --- a/src/test/lexer_svelte.test.ts +++ b/src/test/lexer_svelte.test.ts @@ -248,6 +248,26 @@ describe('lexer_svelte regions', () => { }); }); +describe('lexer_svelte malformed-input resilience', () => { + // a `|` before the `:` in an attribute name is malformed, but must still + // produce a valid (non-overlapping) event stream — the pipe belongs to the + // namespace span, not a separate modifier punctuation leaf + test('a pipe before the namespace colon stays valid', () => { + assert.deepEqual( + validate_syntax_events(syntax_styler_global.lex('
    ', 'svelte')), + [], + ); + }); + + // `@`/`#`/`$` at an embedded ts window's edge (the each/await `as`/`then` + // split boundary) must not read or emit past the window + test('an at-sign at an embed boundary stays valid', () => { + for (const text of ['{#each a@as b}', '{#await x@then y}', '{#each a#as b}']) { + assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(text, 'svelte')), [], text); + } + }); +}); + describe('lexer_svelte sample', () => { const content = readFileSync('src/test/fixtures/samples/sample_complex.svelte', 'utf8'); diff --git a/src/test/pathological.ts b/src/test/pathological.ts index c102d4ea..f4d8b199 100644 --- a/src/test/pathological.ts +++ b/src/test/pathological.ts @@ -130,6 +130,20 @@ export const PATHOLOGICAL_CASES: Array = [ lang: 'md', generate: (size) => repeat_to_size('```ts\nlet x = 1;\n```\n', size), }, + { + // `[`-dense text with no closing `]` — each `[` probes for a link closer + // that never comes; a naive per-`[` forward scan would be quadratic + name: 'md_bracket_dense', + lang: 'md', + generate: (size) => repeat_to_size('[', size), + }, + { + // `[x](`-dense text — exercises both the `]` and `)` link-closer probes + // with no closing `)` in reach + name: 'md_link_paren_dense', + lang: 'md', + generate: (size) => repeat_to_size('[x](', size), + }, { // expression-valued attributes — tag scanner + expression interplay name: 'svelte_attr_expr_dense', From e1bcde254ef120911a4ecdd6cde0f474e9cee708 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 14:19:30 -0400 Subject: [PATCH 08/25] wip --- CLAUDE.md | 115 ++-- README.md | 104 ++- benchmark/compare/compare.ts | 3 +- benchmark/results.md | 61 +- src/lib/Code.svelte | 40 +- src/lib/CodeHighlight.svelte | 31 +- src/lib/CodeTextarea.svelte | 6 +- src/lib/grammar_bash.ts | 175 ----- src/lib/grammar_clike.ts | 48 -- src/lib/grammar_css.ts | 84 --- src/lib/grammar_js.ts | 215 ------ src/lib/grammar_json.ts | 38 -- src/lib/grammar_markdown.ts | 290 --------- src/lib/grammar_markup.ts | 226 ------- src/lib/grammar_svelte.ts | 165 ----- src/lib/grammar_ts.ts | 114 ---- src/lib/highlight_manager.ts | 135 ++-- src/lib/lexer.ts | 3 +- src/lib/range_highlighting.svelte.ts | 33 +- src/lib/svelte_preprocess_fuz_code.ts | 10 +- src/lib/syntax_styler.ts | 682 ++----------------- src/lib/syntax_styler_global.ts | 40 +- src/lib/syntax_token.ts | 49 -- src/lib/tokenize_syntax.ts | 270 -------- src/routes/CodeTome.svelte | 12 +- src/test/fixtures/helpers.ts | 76 +-- src/test/highlight_manager.test.ts | 641 ++++++------------ src/test/svelte_preprocess_fuz_code.test.ts | 4 +- src/test/syntax_styler.stringify.test.ts | 148 ----- src/test/syntax_styler.test.ts | 686 -------------------- 30 files changed, 497 insertions(+), 4007 deletions(-) delete mode 100644 src/lib/grammar_bash.ts delete mode 100644 src/lib/grammar_clike.ts delete mode 100644 src/lib/grammar_css.ts delete mode 100644 src/lib/grammar_js.ts delete mode 100644 src/lib/grammar_json.ts delete mode 100644 src/lib/grammar_markdown.ts delete mode 100644 src/lib/grammar_markup.ts delete mode 100644 src/lib/grammar_svelte.ts delete mode 100644 src/lib/grammar_ts.ts delete mode 100644 src/lib/syntax_token.ts delete mode 100644 src/lib/tokenize_syntax.ts delete mode 100644 src/test/syntax_styler.stringify.test.ts delete mode 100644 src/test/syntax_styler.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 045394f2..71d9ff11 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,10 +1,12 @@ # fuz_code -> Syntax highlighting - a modernized PrismJS fork +> Syntax highlighting - hand-written single-pass lexers fuz_code (`@fuzdev/fuz_code`) is a runtime syntax highlighting library optimized -for HTML generation with CSS classes. It's a PrismJS fork with TypeScript types -and modern module support. +for HTML generation with CSS classes. It originated as a PrismJS fork and keeps +its `.token_*` class vocabulary, but the tokenizer is a full rewrite — one +hand-written single-pass lexer per language emitting a flat token event stream, +with zero regular expressions. For coding conventions, see Skill(fuz-stack). @@ -42,9 +44,9 @@ dev server. fuz_code is a **syntax highlighting library**: - Runtime HTML generation with CSS classes -- PrismJS-compatible grammar definitions +- Hand-written single-pass lexers (zero regex), one per language - 8 built-in languages (TS, JS, CSS, HTML, JSON, Svelte, Markdown, Bash) -- Extensible grammar system with hooks +- Extensible by writing a lexer (`SyntaxLang`) - Optional Svelte component (`Code.svelte`) ### What fuz_code does NOT include @@ -65,13 +67,10 @@ benchmark/ # performance testing └── compare/ # Prism/Shiki comparison src/ ├── lib/ # exportable library code -│ ├── syntax_styler.ts # SyntaxStyler class, hook system +│ ├── syntax_styler.ts # SyntaxStyler class: registry + lex/stylize facade │ ├── syntax_styler_global.ts # pre-configured global instance -│ ├── lexer.ts # lexer-engine substrate: Lexer, TokenTypeRegistry, flat events, HTML render +│ ├── lexer.ts # lexer substrate: Lexer, TokenTypeRegistry, flat events, HTML render │ ├── lexer_*.ts # hand-written lexers (json, ts, css, bash, markup, svelte, md) -│ ├── tokenize_syntax.ts # tokenize_syntax() function (regex engine) -│ ├── syntax_token.ts # SyntaxToken class, type definitions -│ ├── grammar_*.ts # regex language definitions (8 files) │ ├── Code.svelte # main Svelte component │ ├── CodeHighlight.svelte # experimental CSS Highlight API │ ├── highlight_manager.ts # CSS Highlight API manager @@ -80,7 +79,6 @@ src/ │ ├── theme_variables.css # CSS variable fallbacks │ └── theme_highlight.css # CSS Highlight API theme ├── test/ # test files and fixtures -│ ├── syntax_styler.test.ts │ ├── highlight_manager.test.ts │ ├── lexer*.test.ts # lexer-engine suites (substrate + per language) │ ├── lexer.pathological.test.ts # linearity + validity on adversarial inputs @@ -98,43 +96,30 @@ src/ ### Core system -Two engines currently coexist (the lexer engine is replacing the regex -engine, language by language, on the `lexer-architecture` branch): - **Lexer engine** (`lexer.ts` + `lexer_*.ts`) - hand-written single-pass lexers emitting flat token events (`Int32Array`) rendered to HTML in one -forward pass. All 8 built-in languages are ported — `stylize` routes every -registered language through it. Token types intern into a -`TokenTypeRegistry` (`token_types_global` by default, injectable via -`SyntaxStylerOptions`). - -**Regex engine** (`tokenize_syntax.ts` + `grammar_*.ts`) - PrismJS-inherited -multi-pass tokenization; now serves only the legacy `tokenize()` API and -explicit-grammar `stylize` calls, until its deletion completes the rewrite. +forward pass. Token types intern into a `TokenTypeRegistry` +(`token_types_global` by default, injectable via `SyntaxStylerOptions`). -**SyntaxStyler** - The main class for tokenization and HTML generation, -fronting both engines. +**SyntaxStyler** - The main class: a language registry and a `lex`/`stylize` +facade over the lexer engine. `lex(text, lang)` returns the flat event stream +(`LexedSyntax`); `stylize(text, lang)` renders it to HTML. **syntax_styler_global** - Pre-configured instance with all built-in languages registered. Import this for typical usage. ### Token structure -Lexer engine: a flat event stream (`LexedSyntax`) — leaf/open/close records +The lexer emits a flat event stream (`LexedSyntax`) — leaf/open/close records in one `Int32Array` with interned type ids; plain text is implicit between -events. The regex engine builds a hierarchical `SyntaxToken` tree instead: - -- `type` - token type (e.g., 'keyword', 'string') -- `content` - text or nested `SyntaxTokenStream` -- `alias` - CSS class aliases -- `length` - token text length - -Both engines generate HTML using classes like `.token_keyword`, -`.token_string`, styled by `theme.css`. +events (recovered from offsets). `render_syntax_html` streams HTML from it in +one forward pass, wrapping spans with classes like `.token_keyword`, +`.token_string` (styled by `theme.css`); `syntax_events_to_tokens` flattens it +to `{type, start, end}` for tests and fixtures. ### Language definitions -Lexer engine (preferred by `stylize` when registered): +One `SyntaxLang` lexer per language, registered via `add_lang`: - `lexer_json.ts` - JSON (with comments) - `lexer_ts.ts` - TypeScript; also registers the `js`/`javascript` aliases @@ -153,28 +138,9 @@ Lexer engine (preferred by `stylize` when registered): with a per-block inline scan (emphasis, inline code, links, entities, raw markup via the markup scanner); fences embed their languages -Regex grammars (unported languages + `tokenize()` until cutover): - -- `grammar_clike.ts` - base for C-like languages -- `grammar_js.ts` - JavaScript -- `grammar_ts.ts` - TypeScript (extends JS) -- `grammar_css.ts` - CSS stylesheets -- `grammar_markup.ts` - HTML/XML -- `grammar_json.ts` - JSON -- `grammar_svelte.ts` - Svelte components (extends markup) -- `grammar_bash.ts` - Bash/shell -- `grammar_markdown.ts` - Markdown - -### Hook system - -SyntaxStyler provides hooks for customizing tokenization and rendering: - -- **before_tokenize** - modify code/grammar before tokenization -- **after_tokenize** - modify token stream after tokenization -- **wrap** - customize HTML output per token (add attributes, custom wrapping) - -Register with `add_hook_before_tokenize()`, `add_hook_after_tokenize()`, -`add_hook_wrap()`. +Embedded languages resolve lazily by name through the registry (markdown +fences → any language, markup ` diff --git a/src/lib/CodeHighlight.svelte b/src/lib/CodeHighlight.svelte index 73ddabe1..d1fe3c4a 100644 --- a/src/lib/CodeHighlight.svelte +++ b/src/lib/CodeHighlight.svelte @@ -10,7 +10,7 @@ import type {SvelteHTMLElements} from 'svelte/elements'; import {syntax_styler_global} from './syntax_styler_global.ts'; - import type {SyntaxStyler, SyntaxGrammar} from './syntax_styler.ts'; + import type {SyntaxStyler} from './syntax_styler.ts'; import {supports_css_highlight_api, type HighlightMode} from './highlight_manager.ts'; import {create_range_highlighting} from './range_highlighting.svelte.ts'; @@ -18,7 +18,6 @@ content, lang = 'svelte', mode = 'auto', - grammar, inline = false, wrap = false, syntax_styler = syntax_styler_global, @@ -31,17 +30,13 @@ * Language identifier (e.g., 'ts', 'css', 'html', 'json', 'svelte', 'md'). * * **Purpose:** - * - When `grammar` is not provided, used to look up the grammar via `syntax_styler.get_lang(lang)` - * - Used for metadata: sets the `data-lang` attribute and determines `language_supported` + * - Selects the registered lexer used to highlight `content` + * - Sets the `data-lang` attribute and determines `language_supported` * * **Special values:** * - `null` - Explicitly disables syntax highlighting (content rendered as plain text) * - `undefined` - Falls back to default ('svelte') * - * **Relationship with `grammar`:** - * - If both `lang` and `grammar` are provided, `grammar` takes precedence for tokenization - * - However, `lang` is still used for the `data-lang` attribute and language detection - * * @default 'svelte' */ lang?: string | null; @@ -59,23 +54,6 @@ * @default 'auto' */ mode?: HighlightMode; - /** - * Optional custom `SyntaxGrammar` object for syntax tokenization. - * - * **When to use:** - * - To provide a custom language definition not registered in `syntax_styler.langs` - * - To use a modified/extended version of an existing grammar - * - For one-off grammar variations without registering globally - * - * **Behavior:** - * - When provided, this grammar is used for tokenization instead of looking up via `lang` - * - Enables highlighting even if `lang` is not in the registry (useful for custom languages) - * - The `lang` parameter is still used for metadata (data-lang attribute) - * - When undefined, the grammar is automatically looked up via `syntax_styler.get_lang(lang)` - * - * @default undefined (uses grammar from `syntax_styler.langs[lang]`) - */ - grammar?: SyntaxGrammar | undefined; /** * Whether to render as inline code or block code. * Controls display via CSS classes. @@ -123,7 +101,6 @@ text: () => content, enabled: () => use_ranges, lang: () => lang, - grammar: () => grammar, syntax_styler: () => syntax_styler, dev_label: 'CodeHighlight', }); @@ -134,7 +111,7 @@ return ''; } - return syntax_styler.stylize(content, lang!, grammar); // ! is safe bc of the `highlighting_disabled` calculation + return syntax_styler.stylize(content, lang!); // ! is safe bc of the `highlighting_disabled` calculation }); // TODO do syntax styling at compile-time in the normal case, and don't import these at runtime diff --git a/src/lib/CodeTextarea.svelte b/src/lib/CodeTextarea.svelte index 222a4f07..ed54d47c 100644 --- a/src/lib/CodeTextarea.svelte +++ b/src/lib/CodeTextarea.svelte @@ -18,13 +18,12 @@ import type {SvelteHTMLElements} from 'svelte/elements'; import {syntax_styler_global} from './syntax_styler_global.ts'; - import type {SyntaxStyler, SyntaxGrammar} from './syntax_styler.ts'; + import type {SyntaxStyler} from './syntax_styler.ts'; import {create_range_highlighting} from './range_highlighting.svelte.ts'; let { value = $bindable(''), lang = 'svelte', - grammar, syntax_styler = syntax_styler_global, wrapper_attrs, ...rest @@ -36,8 +35,6 @@ * highlighting; `undefined` falls back to the default ('svelte'). */ lang?: string | null; - /** Optional custom grammar; takes precedence over `lang` for tokenization. */ - grammar?: SyntaxGrammar | undefined; /** Custom `SyntaxStyler` instance (defaults to the global one). */ syntax_styler?: SyntaxStyler; /** @@ -66,7 +63,6 @@ element: () => backdrop, text: () => display_text, lang: () => lang, - grammar: () => grammar, syntax_styler: () => syntax_styler, dev_label: 'CodeTextarea', }); diff --git a/src/lib/grammar_bash.ts b/src/lib/grammar_bash.ts deleted file mode 100644 index e4ad1187..00000000 --- a/src/lib/grammar_bash.ts +++ /dev/null @@ -1,175 +0,0 @@ -import type {AddSyntaxGrammar, SyntaxGrammarRaw} from './syntax_styler.ts'; - -/** - * Bash/shell grammar for syntax highlighting. - * - * Standalone grammar (no base extension). Covers core bash syntax: - * comments, strings, variables, functions, keywords, builtins, - * operators, and redirections. - * - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_bash: AddSyntaxGrammar = (syntax_styler) => { - // Shared inside grammar for command substitution — `rest` wired after construction - const command_sub_inside: SyntaxGrammarRaw = { - punctuation: /^\$\(|\)$/, - }; - - // Reusable balanced-paren pattern for $(...) — handles 2 levels of inner () nesting, - // which supports up to 3 levels of $() command substitution (4+ is vanishingly rare) - const command_sub_pattern = /\$\((?:[^()]*|\((?:[^()]*|\([^()]*\))*\))*\)/; - - const grammar_bash = { - // Shebang at file start — matched before general comments - shebang: { - pattern: /^#!.*/, - alias: 'comment', - }, - - // Line comments — require whitespace or start-of-string before # - comment: { - pattern: /(^|\s)#.*/, - lookbehind: true, - greedy: true, - }, - - // Here-documents — must precede string to avoid delimiter consumption - heredoc: [ - // Quoted delimiter (<<'DELIM' or <<"DELIM") — no expansion - { - pattern: /(^|[^<])<<-?\s*(?:['"])(\w+)(?:['"])[\t ]*\n[\s\S]*?\n[\t ]*\2(?=\s*$)/m, - lookbehind: true, - greedy: true, - alias: 'string', - inside: { - // No `m` flag — `^` matches start-of-string (opening) and `$` matches - // end-of-string (closing), so single-word content lines can't false-positive - heredoc_delimiter: [ - { - pattern: /^<<-?\s*(?:['"])\w+(?:['"])/, - alias: 'punctuation', - }, - { - pattern: /\w+$/, - alias: 'punctuation', - }, - ], - } as SyntaxGrammarRaw, - }, - // Unquoted delimiter (<>?|<)/, - alias: 'important', - }, - - // Numbers: hex, octal, base-N, decimal - number: /\b(?:0x[\da-fA-F]+|0[0-7]+|\d+#[\da-zA-Z]+|\d+)\b/, - - // Operators — longest first - operator: /\|\||&&|;;|&>>?|<<>?|=~|[!=]=|[<>]|[|&!]/, - - // Punctuation - punctuation: /[{}[\]();,]/, - } satisfies SyntaxGrammarRaw; - - // Wire circular reference so command substitutions get full bash highlighting - command_sub_inside.rest = grammar_bash; - - syntax_styler.add_lang('bash', grammar_bash, ['sh', 'shell']); -}; diff --git a/src/lib/grammar_clike.ts b/src/lib/grammar_clike.ts deleted file mode 100644 index 1ab09963..00000000 --- a/src/lib/grammar_clike.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type {AddSyntaxGrammar, SyntaxGrammarRaw} from './syntax_styler.ts'; - -export const class_keywords = 'class|extends|implements|instanceof|interface|new'; - -/** - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_clike: AddSyntaxGrammar = (syntax_styler) => { - const grammar_clike = { - comment: [ - { - pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/, - lookbehind: true, - greedy: true, - }, - { - pattern: /(^|[^\\:])\/\/.*/, - lookbehind: true, - greedy: true, - }, - ], - string: { - pattern: /(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/, - greedy: true, - }, - class_name: { - pattern: new RegExp(`(\\b(?:${class_keywords}|trait)\\s+|\\bcatch\\s+\\()[\\w.\\\\]+`, 'i'), - lookbehind: true, - inside: { - punctuation: /[.\\]/, - }, - }, - keyword: - /\b(?:break|catch|continue|do|else|finally|for|function|if|in|instanceof|new|null|return|throw|try|while)\b/, - boolean: /\b(?:false|true)\b/, - function: /\b\w+(?=\()/, - number: /\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i, - operator: /[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/, - punctuation: /[{}[\];(),.:]/, - } satisfies SyntaxGrammarRaw; - - syntax_styler.add_lang('clike', grammar_clike); -}; diff --git a/src/lib/grammar_css.ts b/src/lib/grammar_css.ts deleted file mode 100644 index 837a9f4d..00000000 --- a/src/lib/grammar_css.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type {AddSyntaxGrammar, SyntaxGrammarRaw} from './syntax_styler.ts'; -import {grammar_markup_add_attribute, grammar_markup_add_inlined} from './grammar_markup.ts'; - -var string = /(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/; - -/** - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_css: AddSyntaxGrammar = (syntax_styler) => { - const grammar_css = { - comment: /\/\*[\s\S]*?\*\//, - atrule: { - pattern: RegExp( - '@[\\w-](?:' + - /[^;{\s"']|\s+(?!\s)/.source + - '|' + - string.source + - ')*?' + - /(?:;|(?=\s*\{))/.source, - ), - inside: { - rule: /^@[\w-]+/, - selector_function_argument: { - pattern: - /(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/, - lookbehind: true, - alias: 'selector', - }, - keyword: { - pattern: /(^|[^\w-])(?:and|not|only|or)(?![\w-])/, - lookbehind: true, - }, - } as SyntaxGrammarRaw, // see `rest` below - }, - url: { - // https://drafts.csswg.org/css-values-3/#urls - pattern: RegExp( - '\\burl\\((?:' + string.source + '|' + /(?:[^\\\r\n()"']|\\[\s\S])*/.source + ')\\)', - 'i', - ), - greedy: true, - inside: { - function: /^url/i, - punctuation: /^\(|\)$/, - string: { - pattern: RegExp('^' + string.source + '$'), - alias: 'url', - }, - }, - }, - selector: { - pattern: RegExp( - '(^|[{}\\s])[^{}\\s](?:[^{};"\'\\s]|\\s+(?![\\s{])|' + string.source + ')*(?=\\s*\\{)', - ), - lookbehind: true, - }, - string: { - pattern: string, - greedy: true, - }, - property: { - pattern: /(^|[^-\w\xA0-\uFFFF])(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i, - lookbehind: true, - }, - important: /!important\b/i, - function: { - pattern: /(^|[^-a-z0-9])[-a-z0-9]+(?=\()/i, - lookbehind: true, - }, - punctuation: /[(){};:,]/, - } satisfies SyntaxGrammarRaw; - - grammar_css.atrule.inside.rest = grammar_css; - - syntax_styler.add_lang('css', grammar_css); - - grammar_markup_add_inlined(syntax_styler, 'style', 'css'); - grammar_markup_add_attribute(syntax_styler, 'style', 'css'); -}; diff --git a/src/lib/grammar_js.ts b/src/lib/grammar_js.ts deleted file mode 100644 index 509baa79..00000000 --- a/src/lib/grammar_js.ts +++ /dev/null @@ -1,215 +0,0 @@ -import type {AddSyntaxGrammar} from './syntax_styler.ts'; -import {grammar_markup_add_attribute, grammar_markup_add_inlined} from './grammar_markup.ts'; -import {class_keywords} from './grammar_clike.ts'; - -/** - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_js: AddSyntaxGrammar = (syntax_styler) => { - const grammar_clike = syntax_styler.get_lang('clike'); - - // Main JS keywords (from keyword pattern, excluding those with special lookaheads) - const main_keywords = - 'class|const|debugger|delete|enum|extends|function|implements|in|instanceof|interface|let|new|null|of|package|private|protected|public|static|super|this|typeof|undefined|var|void|with'; - - // Keywords that are treated specially (inserted before 'function') - const special_keywords = - 'as|await|break|case|catch|continue|default|do|else|export|finally|for|from|if|import|return|switch|throw|try|while|yield'; - - // All JS keywords (for negative lookahead in parameter pattern) - // Note: 'assert', 'async', 'get', 'set' have special lookahead requirements in the main keyword pattern - const all_js_keywords = `assert|async|${main_keywords}|get|set|${special_keywords}`; - - const grammar_js = syntax_styler.add_extended_lang( - 'clike', - 'js', - { - class_name: [ - ...grammar_clike.class_name!, - { - pattern: - /(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:constructor|prototype))/, - lookbehind: true, - }, - ], - keyword: [ - { - pattern: new RegExp( - `(^|[^.]|\\.\\.\\.\\s*)\\b(?:assert(?=\\s*\\{)|async(?=\\s*(?:function\\b|\\*|\\(|[$\\w\\xA0-\\uFFFF]|$))|${main_keywords}|(?:get|set)(?=\\s*(?:[#[$\\w\\xA0-\\uFFFF]|$)))\\b`, - ), - lookbehind: true, - }, - ], - // Allow for all non-ASCII characters (See http://stackoverflow.com/a/2008444) - function: - /#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/, - number: { - pattern: RegExp( - /(^|[^\w$])/.source + - '(?:' + - // constant - (/NaN|Infinity/.source + - '|' + - // binary integer - /0[bB][01]+(?:_[01]+)*n?/.source + - '|' + - // octal integer - /0[oO][0-7]+(?:_[0-7]+)*n?/.source + - '|' + - // hexadecimal integer - /0[xX][\dA-Fa-f]+(?:_[\dA-Fa-f]+)*n?/.source + - '|' + - // decimal bigint - /\d+(?:_\d+)*n/.source + - '|' + - // decimal number (integer or float) but no bigint - /(?:\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\.\d+(?:_\d+)*)(?:[Ee][+-]?\d+(?:_\d+)*)?/ - .source) + - ')' + - /(?![\w$])/.source, - ), - lookbehind: true, - }, - operator: - /--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/, - }, - ['javascript'], - ); - - (grammar_js as any).class_name[0].pattern = new RegExp( - `(\\b(?:${class_keywords})\\s+)[\\w.\\\\]+`, - ); - - syntax_styler.grammar_insert_before('js', 'function', { - special_keyword: new RegExp(`\\b(?:${special_keywords})\\b`), - }); - - syntax_styler.grammar_insert_before('js', 'keyword', { - regex: { - pattern: RegExp( - // lookbehind - /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)/.source + - // Regex pattern: - // There are 2 regex patterns here. The RegExp set notation proposal added support for nested character - // classes if the `v` flag is present. Unfortunately, nested CCs are both context-free and incompatible - // with the only syntax, so we have to define 2 different regex patterns. - /\//.source + - '(?:' + - /(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\[\r\n])+\/[dgimyus]{0,7}/.source + - '|' + - // `v` flag syntax. This supports 3 levels of nested character classes. - /(?:\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.)*\])*\])*\]|\\.|[^/\\[\r\n])+\/[dgimyus]{0,7}v[dgimyus]{0,7}/ - .source + - ')' + - // lookahead - /(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/.source, - ), - lookbehind: true, - greedy: true, - inside: { - regex_source: { - pattern: /^(\/)[\s\S]+(?=\/[a-z]*$)/, - lookbehind: true, - alias: 'lang_regex', - inside: syntax_styler.langs.regex, // TODO use `get_lang` after adding `regex` support - }, - regex_delimiter: /^\/|\/$/, - regex_flags: /^[a-z]+$/, - }, - }, - // Arrow function and function expression variable names - // This must be declared before keyword because we use "function" inside the look-forward - function_variable: { - pattern: - /#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/, - alias: 'function', - }, - parameter: [ - { - pattern: - /(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/, - lookbehind: true, - inside: grammar_js, - }, - { - pattern: - /(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i, - lookbehind: true, - inside: grammar_js, - }, - { - pattern: /(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/, - lookbehind: true, - inside: grammar_js, - }, - { - pattern: new RegExp( - `((?:\\b|\\s|^)(?!(?:${all_js_keywords})(?![$\\w\\xA0-\\uFFFF]))(?:(?!\\s)[_$a-zA-Z\\xA0-\\uFFFF](?:(?!\\s)[$\\w\\xA0-\\uFFFF])*\\s*)\\(\\s*|\\]\\s*\\(\\s*)(?!\\s)(?:[^()\\s]|\\s+(?![\\s)])|\\([^()]*\\))+(?=\\s*\\)\\s*\\{)`, - ), - lookbehind: true, - inside: grammar_js, - }, - ], - constant: /\b[A-Z](?:[A-Z_]|\dx?)*\b/, - // Heuristic: treat capitalized identifiers as class names when not already matched - capitalized_identifier: { - pattern: /\b[A-Z][\w]*\b/, - alias: 'class_name', - }, - }); - - syntax_styler.grammar_insert_before('js', 'string', { - hashbang: { - pattern: /^#!.*/, - greedy: true, - alias: 'comment', - }, - template_string: { - pattern: /`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/, - greedy: true, - inside: { - template_punctuation: { - pattern: /^`|`$/, - alias: 'string', - }, - interpolation: { - pattern: /((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/, - lookbehind: true, - inside: { - interpolation_punctuation: { - pattern: /^\$\{|\}$/, - alias: 'punctuation', - }, - rest: grammar_js as any, // TODO try to fix this type - }, - }, - string: /[\s\S]+/, - }, - }, - string_property: { - pattern: /((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m, - lookbehind: true, - greedy: true, - alias: 'property', - }, - }); - - syntax_styler.grammar_insert_before('js', 'operator', { - literal_property: { - pattern: /((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m, - lookbehind: true, - alias: 'property', - }, - }); - - grammar_markup_add_inlined(syntax_styler, 'script', 'js'); - - // add attribute support for all DOM events (on* attributes) - // https://developer.mozilla.org/en-US/docs/Web/Events#Standard_events - grammar_markup_add_attribute(syntax_styler, 'on\\w+', 'js'); -}; diff --git a/src/lib/grammar_json.ts b/src/lib/grammar_json.ts deleted file mode 100644 index 99683098..00000000 --- a/src/lib/grammar_json.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type {AddSyntaxGrammar, SyntaxGrammarRaw} from './syntax_styler.ts'; - -/** - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_json: AddSyntaxGrammar = (syntax_styler) => { - const grammar_json = { - property: { - pattern: /(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?=\s*:)/, - lookbehind: true, - greedy: true, - }, - string: { - pattern: /(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?!\s*:)/, - lookbehind: true, - greedy: true, - }, - comment: { - pattern: /\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/, - greedy: true, - }, - number: /-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i, - punctuation: /[{}[\],]/, - operator: /:/, - boolean: /\b(?:false|true)\b/, - null: { - pattern: /\bnull\b/, - alias: 'keyword', - }, - } satisfies SyntaxGrammarRaw; - - syntax_styler.add_lang('json', grammar_json); -}; diff --git a/src/lib/grammar_markdown.ts b/src/lib/grammar_markdown.ts deleted file mode 100644 index 6a4e4ca2..00000000 --- a/src/lib/grammar_markdown.ts +++ /dev/null @@ -1,290 +0,0 @@ -import type { - AddSyntaxGrammar, - SyntaxGrammarTokenRaw, - SyntaxGrammarValueRaw, - SyntaxStyler, -} from './syntax_styler.ts'; - -interface LangDefinition { - aliases: Array; - id: string; -} - -interface FenceType { - backticks: string; - suffix: string; -} - -/** - * Helper to create fenced code block pattern for a language. - */ -const create_fence_pattern = ( - backticks: string, - aliases: Array, - lang_id: string, - syntax_styler: SyntaxStyler, -): SyntaxGrammarTokenRaw => { - const aliases_pattern = aliases.join('|'); - const pattern = new RegExp( - `^${backticks}(?:${aliases_pattern})[^\\n\\r]*(?:\\r?\\n|\\r)[\\s\\S]*?^${backticks}$`, - 'm', - ); - const code_fence_pattern = new RegExp(`^${backticks}[^\\n\\r]*|^${backticks}$`, 'm'); - - return { - pattern, - greedy: true, - inside: { - code_fence: { - pattern: code_fence_pattern, - alias: 'punctuation', - }, - [`lang_${lang_id}`]: { - pattern: /[\s\S]+/, - inside: syntax_styler.get_lang(lang_id), - }, - }, - }; -}; - -/** - * Helper to create catch-all fence pattern (unknown languages). - */ -const create_catchall_fence = (backticks: string): SyntaxGrammarTokenRaw => { - const pattern = new RegExp(`^${backticks}[^\\n\\r]*(?:\\r?\\n|\\r)[\\s\\S]*?^${backticks}$`, 'm'); - const code_fence_pattern = new RegExp(`^${backticks}[^\\n\\r]*|^${backticks}$`, 'm'); - - return { - pattern, - greedy: true, - inside: { - code_fence: { - pattern: code_fence_pattern, - alias: 'punctuation', - }, - }, - }; -}; - -/** - * Helper to create md self-reference placeholder pattern. - */ -const create_md_placeholder = (backticks: string): SyntaxGrammarTokenRaw => { - const pattern = new RegExp( - `^${backticks}(?:md|markdown)[^\\n\\r]*(?:\\r?\\n|\\r)[\\s\\S]*?^${backticks}$`, - 'm', - ); - const code_fence_pattern = new RegExp(`^${backticks}[^\\n\\r]*|^${backticks}$`, 'm'); - - return { - pattern, - greedy: true, - inside: { - code_fence: { - pattern: code_fence_pattern, - alias: 'punctuation', - }, - // lang_md will be added after registration - }, - }; -}; - -/** - * Markdown grammar extending markup. - * Supports: headings, fenced code blocks (3/4/5 backticks with nesting), lists, blockquotes, - * bold, italic, strikethrough, inline code, and links. - */ -export const add_grammar_markdown: AddSyntaxGrammar = (syntax_styler) => { - // Language definitions with aliases - const langs: Array = [ - {aliases: ['ts', 'typescript'], id: 'ts'}, - {aliases: ['js', 'javascript'], id: 'js'}, - {aliases: ['css'], id: 'css'}, - {aliases: ['html', 'markup'], id: 'markup'}, - {aliases: ['json'], id: 'json'}, - {aliases: ['svelte'], id: 'svelte'}, - {aliases: ['bash', 'sh', 'shell'], id: 'bash'}, - ]; - - // Fence types: higher counts first (for proper precedence in tokenization) - const fence_types: Array = [ - {backticks: '`````', suffix: '5tick'}, - {backticks: '````', suffix: '4tick'}, - {backticks: '```', suffix: ''}, - ]; - - // Build grammar dynamically - const grammar: Record = {}; - const md_self_refs: Array = []; // Track md patterns for later self-reference - - // Generate fence patterns for each type - for (const {backticks, suffix} of fence_types) { - const token_suffix = suffix ? `_${suffix}` : ''; - - // md pattern (self-reference added after registration) - const md_key = `fenced_code${token_suffix}_md`; - grammar[md_key] = create_md_placeholder(backticks); - md_self_refs.push(md_key); - - // Other language patterns (use first alias as token name for backward compatibility) - for (const {aliases, id} of langs) { - const token_name = aliases[0]; // Use first alias for token name - grammar[`fenced_code${token_suffix}_${token_name}`] = create_fence_pattern( - backticks, - aliases, - id, - syntax_styler, - ); - } - - // Catch-all fence - grammar[`fenced_code${token_suffix}`] = create_catchall_fence(backticks); - } - - // Register markdown grammar first, then add self-references for md fences - const grammar_md = syntax_styler.add_extended_lang( - 'markup', - 'md', - { - ...grammar, - - // Headings (# through ######) - heading: { - pattern: /^#{1,6}\s+.+$/m, - inside: { - punctuation: { - pattern: /^#{1,6}/, - alias: 'heading_punctuation', - }, - }, - }, - - // Blockquotes (> at line start) - blockquote: { - pattern: /^>\s*.+$/m, - inside: { - punctuation: { - pattern: /^>/, - alias: 'blockquote_punctuation', - }, - }, - }, - - // Lists (* or - with any preceding whitespace) - list: { - pattern: /^\s*[-*]\s+.+$/m, - inside: { - punctuation: /^\s*[-*]/, - }, - }, - - // Inline elements - - // Links [text](url) - link: { - pattern: /\[[^[\]\n\r]+\]\([^)\n\r]+\)/, - inside: { - link_text_wrapper: { - pattern: /^\[[^\]]+\]/, - inside: { - punctuation: { - pattern: /^\[|\]$/, - alias: 'link_punctuation', - }, - link_text: /[^[\]]+/, - }, - }, - url_wrapper: { - pattern: /\([^)]+\)$/, - inside: { - punctuation: { - pattern: /^\(|\)$/, - alias: 'link_punctuation', - }, - url: /[^()]+/, - }, - }, - }, - }, - - // Inline code `text` - inline_code: { - pattern: /`[^`\n\r]+`/, - alias: 'code', - inside: { - punctuation: { - pattern: /^`|`$/, - alias: 'code_punctuation', - }, - content: /[^`]+/, - }, - }, - - // Bold **text** or __text__ - bold: [ - { - // **text** - no asterisks inside - pattern: /\*\*[^\s*][^*]*[^\s*]\*\*|\*\*[^\s*]\*\*/, - inside: { - punctuation: /^\*\*|\*\*$/, - }, - }, - { - // __text__ - at word boundaries, no underscores inside - pattern: /\b__[^\s_][^_]*[^\s_]__\b|\b__[^\s_]__\b/, - inside: { - punctuation: /^__|__$/, - }, - }, - ], - - // Italic *text* or _text_ - italic: [ - { - // *text* - no asterisks inside - pattern: /\*[^\s*][^*]*[^\s*]\*|\*[^\s*]\*/, - inside: { - punctuation: /^\*|\*$/, - }, - }, - { - // _text_ - at word boundaries, no underscores inside - pattern: /\b_[^\s_][^_]*[^\s_]_\b|\b_[^\s_]_\b/, - inside: { - punctuation: /^_|_$/, - }, - }, - ], - - // Strikethrough ~~text~~ - strikethrough: { - pattern: /~~[^\s~][^~]*[^\s~]~~|~~[^\s~]~~/, - inside: { - punctuation: /^~~|~~$/, - }, - }, - }, - ['markdown'], - ); - - // Add self-reference for markdown-in-markdown (all fence types) - // This must be done after registration to avoid circular dependency - const lang_md_inside = { - pattern: /[\s\S]+/, - inside: syntax_styler.get_lang('md'), - }; - for (const key of md_self_refs) { - // After normalization, grammar values are arrays of SyntaxGrammarToken - // We need to add lang_md as a normalized array - const patterns = grammar_md[key]!; - // Manually normalize the lang_md pattern we're adding - const lang_md_token = { - pattern: lang_md_inside.pattern, - lookbehind: false, - greedy: false, - alias: [], - inside: lang_md_inside.inside, - }; - patterns[0]!.inside!.lang_md = [lang_md_token]; - } -}; diff --git a/src/lib/grammar_markup.ts b/src/lib/grammar_markup.ts deleted file mode 100644 index 54ec37e8..00000000 --- a/src/lib/grammar_markup.ts +++ /dev/null @@ -1,226 +0,0 @@ -import type { - SyntaxStyler, - AddSyntaxGrammar, - SyntaxGrammarRaw, - SyntaxGrammarToken, - SyntaxGrammar, -} from './syntax_styler.ts'; - -/** - * Based on Prism (https://github.com/PrismJS/prism) - * by Lea Verou (https://lea.verou.me/) - * - * MIT license - * - * @see LICENSE - */ -export const add_grammar_markup: AddSyntaxGrammar = (syntax_styler) => { - const grammar_markup = { - comment: { - pattern: //, - greedy: true, - }, - processing_instruction: { - pattern: /<\?[\s\S]+?\?>/, - greedy: true, - }, - // https://www.w3.org/TR/xml/#NT-doctypedecl - doctype: { - pattern: /]*>/i, - greedy: true, - }, - cdata: { - pattern: //i, - greedy: true, - }, - tag: { - pattern: - /<\/?(?!\d)[^\s>/=$<%]+(?:\s(?:\s*[^\s>/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/, - greedy: true, - inside: { - tag: { - pattern: /^<\/?[^\s>/]+/, - inside: { - punctuation: { - pattern: /^<\/?/, - alias: 'tag_punctuation', - }, - namespace: /^[^\s>/:]+:/, - }, - }, - special_attr: [], - attr_value: { - pattern: /=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/, - inside: { - punctuation: [ - { - pattern: /^=/, - alias: 'attr_equals', - }, - { - pattern: /^(\s*)["']|["']$/, - lookbehind: true, - alias: 'attr_quote', - }, - ], - entity: undefined as any, // see below - }, - }, - punctuation: { - pattern: /\/?>/, - alias: 'tag_punctuation', - }, - attr_name: { - pattern: /[^\s>/]+/, - inside: { - namespace: /^[^\s>/:]+:/, - }, - }, - }, - }, - entity: [ - { - pattern: /&[\da-z]{1,8};/i, - alias: 'named_entity', - }, - /&#x?[\da-f]{1,8};/i, - ], - } satisfies SyntaxGrammarRaw; - - grammar_markup.tag.inside.attr_value.inside.entity = grammar_markup.entity; - - syntax_styler.add_lang('markup', grammar_markup, ['html', 'mathml', 'svg']); - syntax_styler.add_extended_lang('markup', 'xml', {}, ['ssml', 'atom', 'rss']); -}; - -/** - * Adds an inlined language to markup. - * - * An example of an inlined language is CSS with `\n
    {x}
    '; - const result = syntax_styler_global.stylize(code, 'svelte'); - assert.ok(result.includes('tag'), 'Should recognize HTML tags'); - // Note: embedded language support depends on implementation - }); -}); - -describe('lastIndex and position management', () => { - test('lastIndex properly reset for greedy patterns', () => { - const syntax_styler = create_styler_with_grammars(); - - // Test with code that has multiple strings at different positions - const code1 = ' "first" "second"'; - const code2 = '"immediate" "next"'; - - const result1 = syntax_styler.stylize(code1, 'js'); - const result2 = syntax_styler.stylize(code2, 'js'); - - // Both should tokenize correctly despite different starting positions - assert.ok(result1.includes('"first"')); - assert.ok(result1.includes('"second"')); - assert.ok(result2.includes('"immediate"')); - assert.ok(result2.includes('"next"')); - }); - - test('greedy patterns work with strings at various positions', () => { - const syntax_styler = create_styler_with_grammars(); - - // Strings at different positions in the text - const code = ` - "start" - "indented" - const x = "inline"; - "far right" - `; - - const result = syntax_styler.stylize(code, 'js'); - - // All strings should be tokenized regardless of position - const string_count = (result.match(//g) || []).length; - assert.equal(string_count, 4, 'Should tokenize all 4 strings'); - }); -}); - -describe('pattern flag edge cases', () => { - test('patterns with existing global flag not double-processed', () => { - const syntax_styler = new SyntaxStyler(); - - // Store the original patterns BEFORE registration - const patterns_before = { - already_global: /test/g, // Already has g flag - case_insensitive: /test/i, // Has i flag but not g - multi_flag: /test/gim, // Multiple flags including g - }; - - // Add a test grammar with patterns that already have flags - syntax_styler.add_lang('test', { - already_global: { - pattern: patterns_before.already_global, - greedy: true, - }, - case_insensitive: { - pattern: patterns_before.case_insensitive, - greedy: true, - }, - multi_flag: { - pattern: patterns_before.multi_flag, - greedy: true, - }, - }); - - const grammar = syntax_styler.get_lang('test'); - // After normalization, grammar values are arrays and patterns have been normalized - - // Patterns that already had g flag should keep the same RegExp instance - assert.equal( - grammar.already_global![0]!.pattern, - patterns_before.already_global, - 'Already global pattern should not be replaced', - ); - assert.equal( - grammar.multi_flag![0]!.pattern, - patterns_before.multi_flag, - 'Multi-flag pattern should not be replaced', - ); - - // Pattern without g gets a new RegExp with g added during normalization - assert.notEqual( - grammar.case_insensitive![0]!.pattern, - patterns_before.case_insensitive, - 'Pattern without g gets replaced with global version during normalization', - ); - assert.equal( - grammar.case_insensitive![0]!.pattern.flags, - 'gi', // Now has both flags - 'Pattern gains global flag', - ); - - // After tokenization, patterns remain unchanged (no runtime mutation) - syntax_styler.stylize('test TEST tEsT', 'test'); - - assert.equal( - grammar.already_global![0]!.pattern, - patterns_before.already_global, - 'Pattern unchanged after tokenization', - ); - }); - - test('non-greedy patterns remain untouched', () => { - const syntax_styler = create_styler_with_grammars(); - const grammar = syntax_styler.get_lang('js'); - - // Get non-greedy patterns (keyword is an array of patterns) - const keyword_patterns = grammar.keyword as Array; - const original_first = keyword_patterns[0]; - const original_second = keyword_patterns[1]; - - syntax_styler.stylize('const let var function class extends', 'js'); - - // Non-greedy patterns should remain exactly the same - assert.equal(keyword_patterns[0], original_first, 'First keyword pattern unchanged'); - assert.equal(keyword_patterns[1], original_second, 'Second keyword pattern unchanged'); - }); -}); - -describe('multiple SyntaxStyler instances', () => { - test('separate instances have independent grammars', () => { - const styler1 = new SyntaxStyler(); - const styler2 = new SyntaxStyler(); - - // Load grammars into both - add_grammar_markup(styler1); - add_grammar_clike(styler1); - add_grammar_js(styler1); - - add_grammar_markup(styler2); - add_grammar_clike(styler2); - add_grammar_js(styler2); - - const grammar1 = styler1.get_lang('js'); - const grammar2 = styler2.get_lang('js'); - - // Store patterns after normalization (grammar values are arrays) - const pattern1 = grammar1.string![0]!.pattern; - const pattern2 = grammar2.string![0]!.pattern; - - // Patterns should be different objects (each instance has its own) - assert.notEqual(pattern1, pattern2, 'Each instance has separate pattern objects'); - - // Verify both patterns already have the global flag from normalization - assert.ok(pattern1.flags.includes('g'), 'styler1 pattern has g flag from normalization'); - assert.ok(pattern2.flags.includes('g'), 'styler2 pattern has g flag from normalization'); - - // Tokenizing with one shouldn't affect the other - styler1.stylize('"test"', 'js'); - styler2.stylize('"test"', 'js'); - - // Patterns remain unchanged after tokenization (no runtime mutation) - assert.equal( - grammar1.string![0]!.pattern, - pattern1, - 'styler1 pattern unchanged after tokenization', - ); - assert.equal( - grammar2.string![0]!.pattern, - pattern2, - 'styler2 pattern unchanged after tokenization', - ); - - // Each instance still has its own pattern object - assert.notEqual( - grammar1.string![0]!.pattern, - grammar2.string![0]!.pattern, - 'Instances remain independent', - ); - }); -}); - -describe('nested tokenization', () => { - test('template strings with nested interpolation', () => { - const syntax_styler = create_styler_with_grammars(); - - // Deeply nested template strings - const code = '`outer ${`inner ${`deepest ${x}`} middle`} end`'; - const result = syntax_styler.stylize(code, 'js'); - - // Should handle all levels of nesting - assert.ok(result.includes('template'), 'Should recognize template strings'); - assert.ok(result.includes('interpolation'), 'Should recognize interpolations'); - - // Count interpolations (should be 3) - const interpolation_count = (result.match(/interpolation/g) || []).length; - assert.ok(interpolation_count >= 3, 'Should handle all nested interpolations'); - }); - - test('regex pattern with special characters', () => { - const syntax_styler = create_styler_with_grammars(); - - // Complex regex that might break if caching is wrong - const code = 'const r = /\\$\\{[^}]+\\}/g;'; // Regex that looks like template syntax - const result = syntax_styler.stylize(code, 'js'); - - assert.ok(result.includes('regex'), 'Should recognize as regex'); - assert.ok(!result.includes('template'), 'Should not confuse with template string'); - }); -}); - -describe('tokenization consistency', () => { - test('same pattern used multiple times in one tokenization', () => { - const syntax_styler = create_styler_with_grammars(); - - // Many strings to ensure pattern is reused - const code = '"a" + "b" + "c" + "d" + "e" + "f" + "g" + "h"'; - const result = syntax_styler.stylize(code, 'js'); - - // All strings should be tokenized - const string_count = (result.match(//g) || []).length; - assert.equal(string_count, 8, 'Should tokenize all 8 strings'); - }); - - test('empty strings handled correctly', () => { - const syntax_styler = create_styler_with_grammars(); - - const code = 'const empty = "";'; - const result = syntax_styler.stylize(code, 'js'); - - assert.ok(result.includes('string'), 'Should tokenize empty string'); - assert.ok(result.includes('""'), 'Should preserve empty string content'); - }); - - test('very long strings handled correctly', () => { - const syntax_styler = create_styler_with_grammars(); - - // Create a very long string to test lastIndex with large values - const long_content = 'x'.repeat(10000); - const code = `const long = "${long_content}";`; - const result = syntax_styler.stylize(code, 'js'); - - assert.ok(result.includes('string'), 'Should tokenize long string'); - assert.ok(result.includes(long_content), 'Should preserve long string content'); - }); -}); - -describe('tokenize (hooked tokenization)', () => { - test('matches the bare tokenize_syntax stream when no hooks are registered', () => { - const syntax_styler = create_styler_with_grammars(); - const code = 'const x = 1;'; - assert.deepEqual( - syntax_styler.tokenize(code, 'js'), - tokenize_syntax(code, syntax_styler.get_lang('js')), - ); - }); - - test('runs before_tokenize and after_tokenize hooks (which bare tokenize_syntax skips)', () => { - const syntax_styler = create_styler_with_grammars(); - let before_lang: string | undefined; - let after_token_count: number | undefined; - syntax_styler.add_hook_before_tokenize((ctx) => { - before_lang = ctx.lang; - }); - syntax_styler.add_hook_after_tokenize((ctx) => { - after_token_count = ctx.tokens.length; - }); - - const tokens = syntax_styler.tokenize('const x = 1;', 'js'); - - assert.equal(before_lang, 'js'); - assert.equal(after_token_count, tokens.length); - }); - - test('a before_tokenize hook can rewrite the code before tokenization', () => { - const syntax_styler = create_styler_with_grammars(); - syntax_styler.add_hook_before_tokenize((ctx) => { - ctx.code = ctx.code.replace('var', 'const'); - }); - - const tokens = syntax_styler.tokenize('var x = 1;', 'js'); - const keyword = tokens.find((t) => typeof t !== 'string' && t.type === 'keyword'); - assert.ok(keyword && typeof keyword !== 'string'); - assert.equal(keyword.content, 'const'); - }); - - test('an after_tokenize hook can replace the token stream', () => { - const syntax_styler = create_styler_with_grammars(); - syntax_styler.add_hook_after_tokenize((ctx) => { - ctx.tokens = ['replaced']; - }); - assert.deepEqual(syntax_styler.tokenize('const x = 1;', 'js'), ['replaced']); - }); - - test('stylize stringifies with the lang a before_tokenize hook rewrote', () => { - const syntax_styler = create_styler_with_grammars(); - let wrap_lang: string | undefined; - // the rewritten lang must flow through to each token's wrap hook context - syntax_styler.add_hook_before_tokenize((ctx) => { - ctx.lang = 'rewritten'; - }); - syntax_styler.add_hook_wrap((ctx) => { - wrap_lang = ctx.lang; - }); - - syntax_styler.stylize('const x = 1;', 'js'); - assert.equal(wrap_lang, 'rewritten'); - }); -}); - -describe('token_types injection', () => { - test('defaults to the shared global registry', () => { - const syntax_styler = new SyntaxStyler(); - assert.strictEqual(syntax_styler.token_types, token_types_global); - assert.strictEqual(syntax_styler_global.token_types, token_types_global); - }); - - test('an injected registry flows through lex and stylize', () => { - const types = new TokenTypeRegistry(); - const t_word = types.intern('word'); - const syntax_styler = new SyntaxStyler({token_types: types}); - syntax_styler.add_lexer_lang({ - id: 'test_words', - lex: (l) => { - l.leaf(t_word, l.pos, l.end); - l.pos = l.end; - }, - }); - const lexed = syntax_styler.lex('abc', 'test_words'); - assert.strictEqual(lexed.types, types); - assert.strictEqual( - syntax_styler.stylize('abc', 'test_words'), - 'abc', - ); - }); -}); From 97178a644a119dc5f033b1a430022ca9e70cdb92 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 14:30:06 -0400 Subject: [PATCH 09/25] wip --- .changeset/dark-birds-go.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/dark-birds-go.md diff --git a/.changeset/dark-birds-go.md b/.changeset/dark-birds-go.md new file mode 100644 index 00000000..6437ce79 --- /dev/null +++ b/.changeset/dark-birds-go.md @@ -0,0 +1,5 @@ +--- +'@fuzdev/fuz_code': minor +--- + +perf: rewrite to lexer architecture dropping regexp grammars From 04715285f7177094868a2e9e7d315a6a0ec583a6 Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Thu, 9 Jul 2026 15:31:44 -0400 Subject: [PATCH 10/25] wip --- README.md | 8 +- src/lib/Code.svelte | 2 +- src/lib/CodeHighlight.svelte | 2 +- src/lib/lexer.ts | 4 +- src/lib/lexer_bash.ts | 116 ++++++- src/lib/lexer_css.ts | 43 ++- src/lib/lexer_json.ts | 6 +- src/lib/lexer_markup.ts | 48 ++- src/lib/lexer_md.ts | 41 +-- src/lib/lexer_svelte.ts | 34 +- src/lib/lexer_ts.ts | 72 ++-- src/routes/samples/all.ts | 1 + .../fixtures/generated/bash/bash_complex.html | 1 + .../fixtures/generated/bash/bash_complex.txt | 326 +++++++++--------- src/test/fixtures/samples/sample_complex.bash | 1 + src/test/lexer_bash.test.ts | 52 +++ src/test/lexer_css.test.ts | 9 + src/test/lexer_json.test.ts | 15 + src/test/lexer_markup.test.ts | 9 + src/test/lexer_md.test.ts | 9 + src/test/lexer_svelte.test.ts | 9 + src/test/lexer_ts.test.ts | 24 ++ src/test/syntax_styler.test.ts | 56 +++ 23 files changed, 596 insertions(+), 292 deletions(-) create mode 100644 src/test/syntax_styler.test.ts diff --git a/README.md b/README.md index 176a02ff..0ee32291 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ **[code.fuz.dev](https://code.fuz.dev/)** -fuz*code is a runtime syntax highlighter: it turns source code into HTML with -`.token*\_`CSS classes, and knows nothing about the DOM. It originated as a fork +`fuz_code` is a runtime syntax highlighter: it turns source code into HTML with +`.token_*` CSS classes, and knows nothing about the DOM. It originated as a fork of [Prism](https://github.com/PrismJS/prism) ([prismjs.com](https://prismjs.com/)) -and keeps its`.token\_\_` class vocabulary, but the tokenizer is now a full +and keeps its `.token_*` class vocabulary, but the tokenizer is now a full rewrite — one hand-written single-pass lexer per language emitting a flat token event stream, with zero regular expressions. @@ -34,7 +34,7 @@ Compared to [Shiki](https://github.com/shikijs/shiki), fuz_code is much lighter and [vastly faster](./benchmark/compare/results.md) for runtime usage: it runs hand-written single-pass lexers rather than the [Oniguruma regexp engine](https://shiki.matsu.io/guide/regex-engines) that -TextMate grammars require, and has zero dependencies instead of 38. Shiki +TextMate grammars require, and has zero runtime dependencies instead of 38. Shiki targets build-time use and supports far more languages and themes — pick the tool that fits; fuz_code is optimized for small, fast, runtime highlighting. diff --git a/src/lib/Code.svelte b/src/lib/Code.svelte index 9bdccfcf..990b7864 100644 --- a/src/lib/Code.svelte +++ b/src/lib/Code.svelte @@ -79,7 +79,7 @@ nomargin?: boolean; /** * Custom `SyntaxStyler` instance to use for highlighting. - * Allows using a different styler with custom grammars or configuration. + * Allows using a different styler with custom languages or configuration. * * @default syntax_styler_global */ diff --git a/src/lib/CodeHighlight.svelte b/src/lib/CodeHighlight.svelte index d1fe3c4a..9ebb5289 100644 --- a/src/lib/CodeHighlight.svelte +++ b/src/lib/CodeHighlight.svelte @@ -77,7 +77,7 @@ wrap?: boolean; /** * Custom `SyntaxStyler` instance to use for highlighting. - * Allows using a different styler with custom grammars or configuration. + * Allows using a different styler with custom languages or configuration. * * @default syntax_styler_global */ diff --git a/src/lib/lexer.ts b/src/lib/lexer.ts index de8644c5..aeea4037 100644 --- a/src/lib/lexer.ts +++ b/src/lib/lexer.ts @@ -94,7 +94,7 @@ export const token_type = (name: string, alias?: string | Array): number token_types_global.intern(name, alias); /** - * A lexer-based language registration — the replacement for regex grammars. + * A lexer-based language registration. */ export interface SyntaxLang { /** @@ -263,7 +263,7 @@ export const lex_syntax = ( /** * Escapes `text[from..to)` for HTML text content in a single pass. * Only `&`, `<`, and non-breaking spaces need handling (nbsp normalizes to a - * regular space, matching the old engine's policy). + * regular space). */ const escape_html_slice = (text: string, from: number, to: number): string => { let out = ''; diff --git a/src/lib/lexer_bash.ts b/src/lib/lexer_bash.ts index 178fcde3..347a5856 100644 --- a/src/lib/lexer_bash.ts +++ b/src/lib/lexer_bash.ts @@ -11,14 +11,16 @@ import { /** * Hand-written Bash/shell lexer. * - * Token vocabulary matches the retired regex grammar: `shebang`, `comment`, - * `string` (double-quoted strings are containers whose `$`-expansions nest as - * `variable`/`command_substitution`), `keyword`, `builtin`, `boolean`, - * `number`, `variable`, `command_substitution` (a container), `function`, - * `file_descriptor`, `operator`, `punctuation`, plus heredocs (`heredoc` - * container + `heredoc_delimiter`). + * Emits: `shebang`, `comment`, `string` (double-quoted strings are containers + * whose `$`-expansions nest as `variable`/`command_substitution`), `keyword`, + * `builtin`, `boolean`, `number`, `variable`, `command_substitution` (a + * container), `function`, `file_descriptor`, `operator`, `punctuation`, plus + * heredocs (`heredoc` container + `heredoc_delimiter`). * - * Fidelity fixes over the regex model: + * Notable behavior: + * - command substitution is recognized in both `$(…)` and legacy backtick + * form; both are `command_substitution` containers with an ordinary-bash + * interior. * - arithmetic expansion `$((…))` is recognized as distinct from command * substitution `$(…)`; its `$((`/`))` are punctuation and the interior lexes * as ordinary bash (numbers, `$vars`, and general operators fall out @@ -28,7 +30,7 @@ import { * one line are queued and their bodies consumed in order. * * Word classification (keyword/builtin/boolean) is a context-free `Map` - * lookup, matching the old grammar's type-major behavior. + * lookup. * * Resilience: unterminated single-line constructs stop at the line boundary; * unterminated strings, command substitutions, and heredocs extend to the @@ -205,9 +207,17 @@ const scan_dollar_var = (l: Lexer, i: number, to: number): number => { return i + 1; // bare `$` }; +// bounds recursive `$(…)`/`$((…))` nesting so deeply nested substitutions +// degrade to plain interiors instead of overflowing the JS call stack — far +// beyond any nesting real shell reaches. Safe as module state: lexing is +// synchronous and `lex_bash` resets it per top-level document. +const MAX_SUBST_DEPTH = 64; +let subst_depth = 0; + /** * Lexes a `$(…)` command substitution at `i` as a container, returning the new - * position. Interior lexes as ordinary bash; unterminated extends to `to`. + * position. Interior lexes as ordinary bash (bounded by `MAX_SUBST_DEPTH`); + * unterminated extends to `to`. * * @mutates `l` */ @@ -216,12 +226,16 @@ const lex_bash_cmdsub = (l: Lexer, i: number, to: number): number => { l.open(T_COMMAND_SUBSTITUTION, i); l.leaf(T_PUNCTUATION, i, i + 2); // `$(` const close = scan_cmdsub_end(text, i + 2, to); + const inner_end = close === -1 ? to : close; + if (subst_depth < MAX_SUBST_DEPTH) { + subst_depth++; + lex_bash_window(l, i + 2, inner_end); + subst_depth--; + } if (close === -1) { - lex_bash_window(l, i + 2, to); l.close(to); return to; } - lex_bash_window(l, i + 2, close); l.leaf(T_PUNCTUATION, close, close + 1); // `)` l.close(close + 1); return close + 1; @@ -254,18 +268,75 @@ const lex_bash_arith = (l: Lexer, i: number, to: number): number => { } } l.leaf(T_PUNCTUATION, i, i + 3); // `$((` - if (depth === 0) { - lex_bash_window(l, i + 3, j - 1); + const closed = depth === 0; + const inner_end = closed ? j - 1 : to; + if (subst_depth < MAX_SUBST_DEPTH) { + subst_depth++; + lex_bash_window(l, i + 3, inner_end); + subst_depth--; + } + if (closed) { l.leaf(T_PUNCTUATION, j - 1, j + 1); // `))` return j + 1; } - lex_bash_window(l, i + 3, to); // unterminated + return to; // unterminated +}; + +/** + * Dispatches a `$(`-led expansion at `i`: arithmetic `$((…))` when a second + * `(` follows in-window, else command substitution `$(…)`. Returns the new + * position. The `i + 2 < to` guard keeps the lookahead inside the window. + * + * @mutates `l` + */ +const lex_bash_dollar_paren = (l: Lexer, i: number, to: number): number => + i + 2 < to && l.text.charCodeAt(i + 2) === 40 + ? lex_bash_arith(l, i, to) + : lex_bash_cmdsub(l, i, to); + +/** + * Lexes a legacy backtick command substitution at `i` as a container, + * returning the new position. The interior lexes as ordinary bash (bounded by + * `MAX_SUBST_DEPTH`); a backslash escapes the next char; unterminated extends + * to `to`. + * + * @mutates `l` + */ +const lex_bash_backtick = (l: Lexer, i: number, to: number): number => { + const {text} = l; + let j = i + 1; + while (j < to) { + const c = text.charCodeAt(j); + if (c === 92) { + j += 2; // backslash escapes the next char (e.g. an inner backtick) + } else if (c === 96) { + break; + } else { + j++; + } + } + const closed = j < to && text.charCodeAt(j) === 96; + const inner_end = closed ? j : to; + l.open(T_COMMAND_SUBSTITUTION, i); + l.leaf(T_PUNCTUATION, i, i + 1); // opening backtick + if (subst_depth < MAX_SUBST_DEPTH) { + subst_depth++; + lex_bash_window(l, i + 1, inner_end); + subst_depth--; + } + if (closed) { + l.leaf(T_PUNCTUATION, j, j + 1); // closing backtick + l.close(j + 1); + return j + 1; + } + l.close(to); return to; }; /** * Lexes a `"…"` double-quoted string at `i` as a container, returning the new - * position. `$`-expansions nest; unterminated extends to `to`. + * position. `$`-expansions and backtick substitutions nest; unterminated + * extends to `to`. * * @mutates `l` */ @@ -283,10 +354,12 @@ const lex_bash_dquote = (l: Lexer, i: number, to: number): number => { } else if (c === 36) { const c1 = text.charCodeAt(j + 1); if (c1 === 40) { - j = text.charCodeAt(j + 2) === 40 ? lex_bash_arith(l, j, to) : lex_bash_cmdsub(l, j, to); + j = lex_bash_dollar_paren(l, j, to); } else { j = scan_dollar_var(l, j, to); } + } else if (c === 96) { + j = lex_bash_backtick(l, j, to); } else { j++; } @@ -348,7 +421,7 @@ const expand_heredoc_body = (l: Lexer, from: number, to: number): void => { if (text.charCodeAt(j) === 36) { const c1 = text.charCodeAt(j + 1); if (c1 === 40) { - j = text.charCodeAt(j + 2) === 40 ? lex_bash_arith(l, j, to) : lex_bash_cmdsub(l, j, to); + j = lex_bash_dollar_paren(l, j, to); } else { j = scan_dollar_var(l, j, to); } @@ -479,7 +552,7 @@ const lex_bash_window = (l: Lexer, from: number, to: number): void => { if (c === 36) { const c1 = text.charCodeAt(i + 1); if (c1 === 40) { - i = text.charCodeAt(i + 2) === 40 ? lex_bash_arith(l, i, to) : lex_bash_cmdsub(l, i, to); + i = lex_bash_dollar_paren(l, i, to); continue; } if (c1 === 39) { @@ -505,6 +578,12 @@ const lex_bash_window = (l: Lexer, from: number, to: number): void => { continue; } + // legacy backtick command substitution + if (c === 96) { + i = lex_bash_backtick(l, i, to); + continue; + } + // numbers and file descriptors if (is_digit(c)) { const next = text.charCodeAt(i + 1); @@ -697,6 +776,7 @@ const lex_bash_heredoc_start = ( }; const lex_bash = (l: Lexer): void => { + subst_depth = 0; lex_bash_window(l, l.pos, l.end); }; diff --git a/src/lib/lexer_css.ts b/src/lib/lexer_css.ts index 1850c49d..421dc193 100644 --- a/src/lib/lexer_css.ts +++ b/src/lib/lexer_css.ts @@ -3,21 +3,19 @@ import {is_space, matches_ci, token_type, type Lexer, type SyntaxLang} from './l /** * Hand-written CSS lexer. * - * Token vocabulary matches the retired regex grammar: `comment`, `atrule` - * (a container wrapping `rule` + prelude), `rule`, `selector`, `string`, - * `property`, `important`, `function`, `url` (a container), `keyword` - * (`and`/`not`/`only`/`or` inside at-rule preludes), and `punctuation`. + * Emits: `comment`, `atrule` (a container wrapping `rule` + prelude), `rule`, + * `selector`, `string`, `property`, `important`, `function`, `url` (a + * container), `keyword` (`and`/`not`/`only`/`or` inside at-rule preludes), and + * `punctuation`. * - * Structure is decided by a brace-context lookahead rather than the old - * lookahead regexes: from each item, the first top-level `{`, `;`, or `}` - * (skipping strings, comments, `()`, and `[]`) says whether the item is a - * qualified rule (its prelude is a `selector`) or a declaration - * (`property : value`). Because that decision is purely local, the flat main - * loop nests to any depth — native CSS nesting works for free (the agreed - * fidelity extra), where the regex model could not express it. + * Structure is decided by a brace-context lookahead: from each item, the first + * top-level `{`, `;`, or `}` (skipping strings, comments, `()`, and `[]`) says + * whether the item is a qualified rule (its prelude is a `selector`) or a + * declaration (`property : value`). Because that decision is purely local, the + * flat main loop nests to any depth, so native CSS nesting works for free. * - * Values (declaration right-hand sides and at-rule preludes) reproduce the old - * grammar's sparse output: only strings, `url()`, functions, `!important`, and + * Values (declaration right-hand sides and at-rule preludes) are tokenized + * sparsely: only strings, `url()`, functions, `!important`, and * `property`-before-colon are tokenized; numbers, colors, units, and bare * identifiers stay plain text. * @@ -39,7 +37,20 @@ const T_URL = token_type('url'); const T_KEYWORD = token_type('keyword'); // at-rule prelude keywords (`@media screen and (...)`) -const ATRULE_KEYWORDS: Set = new Set(['and', 'not', 'only', 'or']); +// at-rule prelude keywords (`@media screen and (...)`), matched +// case-insensitively without allocating (matching the rest of this file) +const is_atrule_keyword = (text: string, from: number, to: number): boolean => { + switch (to - from) { + case 2: + return matches_ci(text, from, 'or'); + case 3: + return matches_ci(text, from, 'and') || matches_ci(text, from, 'not'); + case 4: + return matches_ci(text, from, 'only'); + default: + return false; + } +}; // a css identifier char: letters, digits, `-`, `_`, and non-ASCII const is_css_ident = (c: number): boolean => @@ -57,7 +68,7 @@ const is_css_ident_start = (c: number): boolean => /** * Scans a `"` or `'` string from `i`, returning the exclusive end. * Handles `\` escapes (including escaped newlines); an unterminated string - * stops at the newline (matching the old grammar's `[^"\\\r\n]` class). + * stops at the newline. */ const scan_css_string = (text: string, from: number, end: number, quote: number): number => { let i = from + 1; @@ -190,7 +201,7 @@ const lex_css_value = (l: Lexer, from: number, to: number, is_atrule: boolean): i = ident_end; continue; } - if (is_atrule && ATRULE_KEYWORDS.has(text.slice(i, ident_end).toLowerCase())) { + if (is_atrule && is_atrule_keyword(text, i, ident_end)) { l.leaf(T_KEYWORD, i, ident_end); i = ident_end; continue; diff --git a/src/lib/lexer_json.ts b/src/lib/lexer_json.ts index a34bb411..b4be178d 100644 --- a/src/lib/lexer_json.ts +++ b/src/lib/lexer_json.ts @@ -12,9 +12,9 @@ import { /** * Hand-written JSON lexer (accepts JSONC — line and block comments). * - * Token vocabulary matches the retired regex grammar: `property` (a string - * key), `string`, `comment`, `number`, `punctuation`, `operator` (`:`), - * `boolean`, and `null` (aliased to `keyword`). + * Emits: `property` (a string key), `string`, `comment`, `number`, + * `punctuation`, `operator` (`:`), `boolean`, and `null` (aliased to + * `keyword`). * * Resilience: unterminated strings extend to end of line; unterminated block * comments extend to end of window. diff --git a/src/lib/lexer_markup.ts b/src/lib/lexer_markup.ts index 1c3af57a..79caf472 100644 --- a/src/lib/lexer_markup.ts +++ b/src/lib/lexer_markup.ts @@ -11,11 +11,10 @@ import { /** * Hand-written HTML/XML lexer. * - * Token vocabulary matches the retired regex grammar: flat `comment`, - * `processing_instruction`, `doctype`, `cdata`, and `entity` - * (alias `named_entity` for the `&`-style form); a `tag` container holding - * a nested `tag` (the `` (the old pattern refused comments + * HTML-semantics details: a comment ends at the first `-->` (even one * containing a second `