', ['special_attr', 'value']),
+ [],
+ );
+ });
+});
+
+describe('lexer_svelte regions', () => {
+ test('script embeds ts', () => {
+ assert.deepEqual(picked('', ['script', 'lang_ts', 'keyword']), [
+ ['script', 'let x = 1;'],
+ ['lang_ts', 'let x = 1;'],
+ ['keyword', 'let'],
+ ]);
+ });
+
+ test('style embeds css', () => {
+ assert.deepEqual(picked('', ['style', 'lang_css', 'property']), [
+ ['style', 'a{color:red}'],
+ ['lang_css', 'a{color:red}'],
+ ['property', 'color'],
+ ]);
+ });
+
+ test('textarea keeps expressions live', () => {
+ assert.deepEqual(picked('
', ['svelte_expression']), [
+ ['svelte_expression', '{value}'],
+ ]);
+ });
+
+ test('comments swallow expressions', () => {
+ assert.deepEqual(tokens_of(''), [['comment', '']]);
+ });
+});
+
+describe('lexer_svelte malformed-input resilience', () => {
+ // a `|` before the `:` in an attribute name is malformed, but must still
+ // produce a valid (non-overlapping) event stream — the pipe belongs to the
+ // namespace span, not a separate modifier punctuation leaf
+ test('a pipe before the namespace colon stays valid', () => {
+ assert.deepEqual(
+ validate_syntax_events(syntax_styler_global.lex('
', 'svelte')),
+ [],
+ );
+ });
+
+ // `@`/`#`/`$` at an embedded ts window's edge (the each/await `as`/`then`
+ // split boundary) must not read or emit past the window
+ test('an at-sign at an embed boundary stays valid', () => {
+ for (const text of ['{#each a@as b}', '{#await x@then y}', '{#each a#as b}']) {
+ assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(text, 'svelte')), [], text);
+ }
+ });
+});
+
+describe('lexer_svelte sample', () => {
+ const content = readFileSync('src/test/fixtures/samples/sample_complex.svelte', 'utf8');
+
+ test('lexes the sample with valid invariants', () => {
+ assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(content, 'svelte')), []);
+ });
+
+ test('sample produces its characteristic token types', () => {
+ const types = new Set(
+ syntax_events_to_tokens(syntax_styler_global.lex(content, 'svelte')).map((t) => t.type),
+ );
+ for (const t of ['tag', 'svelte_expression', 'lang_ts', 'attr_name', 'block', 'at_directive']) {
+ assert.ok(types.has(t), `expected a ${t} token in the sample`);
+ }
+ });
+
+ test('every prefix lexes without throwing, with valid invariants', () => {
+ for (let len = 0; len <= content.length; len += 11) {
+ const prefix = content.slice(0, len);
+ const lexed = syntax_styler_global.lex(prefix, 'svelte');
+ assert.deepEqual(validate_syntax_events(lexed), [], `prefix of length ${len}`);
+ }
+ });
+
+ test('lexing is deterministic', () => {
+ const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'svelte'));
+ const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'svelte'));
+ assert.deepEqual(a, b);
+ });
+});
diff --git a/src/test/lexer_ts.test.ts b/src/test/lexer_ts.test.ts
new file mode 100644
index 00000000..01f3a0d5
--- /dev/null
+++ b/src/test/lexer_ts.test.ts
@@ -0,0 +1,405 @@
+import {describe, test, assert} from 'vitest';
+import {readFileSync} from 'node:fs';
+
+import {syntax_styler_global} from '$lib/syntax_styler_global.ts';
+import {SyntaxStyler} from '$lib/syntax_styler.ts';
+import {lexer_ts} from '$lib/lexer_ts.ts';
+import {
+ syntax_events_to_tokens,
+ token_type,
+ validate_syntax_events,
+ type SyntaxLang,
+} from '$lib/lexer.ts';
+
+/**
+ * Lexes `text` as ts and returns `[type, text]` pairs.
+ * Containers appear before their children with their full span text.
+ */
+const tokens_of = (text: string): Array<[string, string]> =>
+ syntax_events_to_tokens(syntax_styler_global.lex(text, 'ts')).map((t) => [
+ t.type,
+ text.slice(t.start, t.end),
+ ]);
+
+/**
+ * Like `tokens_of` but only `[type, text]` of the named types — for targeted
+ * assertions that ignore surrounding punctuation/operators.
+ */
+const picked = (text: string, types: Array
): Array<[string, string]> =>
+ tokens_of(text).filter(([type]) => types.includes(type));
+
+describe('lexer_ts keywords', () => {
+ test('main and special keywords', () => {
+ assert.deepEqual(picked('if (x) return new Foo();', ['keyword', 'special_keyword']), [
+ ['special_keyword', 'if'],
+ ['special_keyword', 'return'],
+ ['keyword', 'new'],
+ ]);
+ });
+
+ test('keywords after member access are properties, not keywords', () => {
+ assert.deepEqual(picked('a.delete(); b?.new;', ['keyword', 'special_keyword']), []);
+ // and `a.delete()` is a function call
+ assert.deepEqual(picked('a.delete();', ['function']), [['function', 'delete']]);
+ });
+
+ test('contextual keywords', () => {
+ assert.deepEqual(picked('async () => x; const get = 1; get x() {}', ['keyword']), [
+ ['keyword', 'async'],
+ ['keyword', 'const'],
+ ['keyword', 'get'],
+ ]);
+ });
+
+ test('null and undefined are keywords, true/false booleans, NaN a number', () => {
+ assert.deepEqual(tokens_of('null undefined true NaN'), [
+ ['keyword', 'null'],
+ ['keyword', 'undefined'],
+ ['boolean', 'true'],
+ ['number', 'NaN'],
+ ]);
+ });
+});
+
+describe('lexer_ts strings and templates', () => {
+ test('string properties in object literals', () => {
+ assert.deepEqual(picked('{"a": 1, \'b\': 2}', ['string_property', 'string']), [
+ ['string_property', '"a"'],
+ ['string_property', "'b'"],
+ ]);
+ });
+
+ test('ternary strings are not properties', () => {
+ assert.deepEqual(picked('x ? "a" : "b"', ['string_property', 'string']), [
+ ['string', '"a"'],
+ ['string', '"b"'],
+ ]);
+ });
+
+ test('strings containing // are not comments', () => {
+ assert.deepEqual(picked('const a = "x // y";', ['string', 'comment']), [
+ ['string', '"x // y"'],
+ ]);
+ });
+
+ test('template strings with interpolation', () => {
+ const tokens = tokens_of('`a${b}c`');
+ assert.deepEqual(tokens, [
+ ['template_string', '`a${b}c`'],
+ ['template_punctuation', '`'],
+ ['string', 'a'],
+ ['interpolation', '${b}'],
+ ['interpolation_punctuation', '${'],
+ ['interpolation_punctuation', '}'],
+ ['string', 'c'],
+ ['template_punctuation', '`'],
+ ]);
+ });
+
+ test('nested templates in interpolations', () => {
+ const text = '`a${`b${c}`}d`';
+ const lexed = syntax_styler_global.lex(text, 'ts');
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ const types = syntax_events_to_tokens(lexed).map((t) => t.type);
+ assert.strictEqual(types.filter((t) => t === 'template_string').length, 2);
+ });
+
+ test('unterminated template extends to end', () => {
+ const lexed = syntax_styler_global.lex('`abc', 'ts');
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ const tokens = syntax_events_to_tokens(lexed);
+ assert.strictEqual(tokens[0]!.type, 'template_string');
+ assert.strictEqual(tokens[0]!.end, 4);
+ });
+
+ test('nested braces inside an interpolation are depth-tracked', () => {
+ assert.deepEqual(picked('`${ {a: 1} }`', ['interpolation']), [
+ ['interpolation', '${ {a: 1} }'],
+ ]);
+ });
+
+ test('interpolation spans a regex literal containing `}`', () => {
+ // the interpolation's end is discovered during real tokenization, so the
+ // `}` inside the regex body cannot close it early
+ const text = '`${/}/.test(x)}`';
+ assert.deepEqual(validate_syntax_events(syntax_styler_global.lex(text, 'ts')), []);
+ assert.deepEqual(picked(text, ['interpolation', 'regex']), [
+ ['interpolation', '${/}/.test(x)}'],
+ ['regex', '/}/'],
+ ]);
+ });
+
+ test('a malformed interpolation interior propagates to the template end', () => {
+ // an unterminated block comment consumes the closing `}` and backtick,
+ // so the damage extends editor-style instead of being contained
+ const text = '`${/* oops}` and more';
+ const lexed = syntax_styler_global.lex(text, 'ts');
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ const comment = syntax_events_to_tokens(lexed).find((t) => t.type === 'comment');
+ assert.ok(comment);
+ assert.strictEqual(comment.end, text.length);
+ });
+});
+
+describe('lexer_ts regex vs division', () => {
+ test('regex literal in expression position', () => {
+ assert.deepEqual(picked('const re = /ab+c/gi;', ['regex', 'regex_source', 'regex_flags']), [
+ ['regex', '/ab+c/gi'],
+ ['regex_source', 'ab+c'],
+ ['regex_flags', 'gi'],
+ ]);
+ });
+
+ test('division is not a regex', () => {
+ assert.deepEqual(picked('const x = a / b / c;', ['regex']), []);
+ });
+
+ test('regex after return keyword', () => {
+ assert.deepEqual(picked('return /x/;', ['regex']), [['regex', '/x/']]);
+ });
+
+ test('regex with slash in character class', () => {
+ assert.deepEqual(picked('x = /[/]/;', ['regex']), [['regex', '/[/]/']]);
+ });
+});
+
+describe('lexer_ts identifiers', () => {
+ test('class contexts', () => {
+ assert.deepEqual(picked('class Foo extends Bar {}', ['class_name']), [
+ ['class_name', 'Foo'],
+ ['class_name', 'Bar'],
+ ]);
+ assert.deepEqual(picked('new a.b.Thing()', ['class_name']), [
+ ['class_name', 'a'],
+ ['class_name', 'b'],
+ ['class_name', 'Thing'],
+ ]);
+ });
+
+ test('constants and capitalized identifiers', () => {
+ assert.deepEqual(tokens_of('MAX_VALUE Foo'), [
+ ['constant', 'MAX_VALUE'],
+ ['capitalized_identifier', 'Foo'],
+ ]);
+ });
+
+ test('function calls and function-valued variables', () => {
+ assert.deepEqual(picked('foo(); const f = (a) => a;', ['function', 'function_variable']), [
+ ['function', 'foo'],
+ ['function_variable', 'f'],
+ ]);
+ });
+
+ test('generic function calls', () => {
+ const tokens = tokens_of('foo(x)');
+ assert.deepEqual(tokens[0], ['generic_function', 'foo']);
+ assert.deepEqual(picked('foo(x)', ['function', 'type_name']), [
+ ['function', 'foo'],
+ ['type_name', 'Bar'],
+ ]);
+ });
+
+ test('comparisons are not generics', () => {
+ assert.deepEqual(picked('a < b; c > d;', ['generic_function']), []);
+ assert.deepEqual(picked('a < b; c > d;', ['operator']), [
+ ['operator', '<'],
+ ['operator', '>'],
+ ]);
+ });
+
+ test('comparisons across statement or interpolation boundaries are not generics', () => {
+ // an unbalanced `)`/`}`/`]` between `<` and `>` rejects the generic scan
+ assert.deepEqual(picked('if (a < b) { } x > (c);', ['generic_function']), []);
+ const text = '`${a < b} ${c > (d)}`';
+ assert.deepEqual(picked(text, ['generic_function']), []);
+ assert.deepEqual(picked(text, ['interpolation']), [
+ ['interpolation', '${a < b}'],
+ ['interpolation', '${c > (d)}'],
+ ]);
+ });
+
+ test('balanced object and tuple types stay inside generics', () => {
+ assert.deepEqual(picked('foo<{a: B}>(x); bar<[C, D]>(y);', ['generic_function']), [
+ ['generic_function', 'foo<{a: B}>'],
+ ['generic_function', 'bar<[C, D]>'],
+ ]);
+ });
+
+ test('lowercase builtins', () => {
+ assert.deepEqual(picked('console.log(x); const s: unknown = 1;', ['builtin']), [
+ ['builtin', 'console'],
+ ['builtin', 'unknown'], // inside the type annotation region
+ ]);
+ });
+});
+
+describe('lexer_ts type syntax', () => {
+ test('type alias declarations', () => {
+ assert.deepEqual(picked('type Foo = Bar;', ['keyword', 'class_name']), [
+ ['keyword', 'type'],
+ ['class_name', 'Foo'],
+ ]);
+ });
+
+ test('import type', () => {
+ assert.deepEqual(picked("import type {A} from 'b';", ['import_type_keyword']), [
+ ['import_type_keyword', 'type'],
+ ]);
+ });
+
+ test('type assertions with as and satisfies', () => {
+ assert.deepEqual(picked('x as Foo; y satisfies Baz;', ['type_assertion']), [
+ ['type_assertion', 'Foo'],
+ ['type_assertion', 'Baz'],
+ ]);
+ });
+
+ test('type annotations with initializers', () => {
+ const tokens = tokens_of('const x: Map = y;');
+ const annotation = tokens.find(([type]) => type === 'type_annotation');
+ assert.ok(annotation);
+ assert.deepEqual(picked('const x: Map = y;', ['type_name']), [
+ ['type_name', 'Map'],
+ ['type_name', 'Foo'],
+ ]);
+ });
+
+ test('object literal colons are not annotations', () => {
+ assert.deepEqual(picked('const o = {a: b};', ['type_annotation']), []);
+ });
+});
+
+describe('lexer_ts decorators and misc', () => {
+ test('decorators', () => {
+ assert.deepEqual(tokens_of('@component'), [
+ ['decorator', '@component'],
+ ['at', '@'],
+ ['function', 'component'],
+ ]);
+ });
+
+ test('hashbang', () => {
+ assert.deepEqual(tokens_of('#!/usr/bin/env node\nx')[0], ['hashbang', '#!/usr/bin/env node']);
+ });
+
+ test('numeric formats', () => {
+ assert.deepEqual(picked('0xff 0b10 1_000n 1.5e-3 .5', ['number']), [
+ ['number', '0xff'],
+ ['number', '0b10'],
+ ['number', '1_000n'],
+ ['number', '1.5e-3'],
+ ['number', '.5'],
+ ]);
+ });
+
+ test('spread and optional chaining operators', () => {
+ assert.deepEqual(picked('f(...a); b?.c ?? d;', ['operator']), [
+ ['operator', '...'],
+ ['operator', '?.'],
+ ['operator', '??'],
+ ]);
+ });
+});
+
+describe('lexer_ts sample', () => {
+ const content = readFileSync('src/test/fixtures/samples/sample_complex.ts', 'utf8');
+
+ test('sample lexes with valid invariants', () => {
+ const lexed = syntax_styler_global.lex(content, 'ts');
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ });
+
+ test('sample produces its characteristic token types', () => {
+ const types = new Set(
+ syntax_events_to_tokens(syntax_styler_global.lex(content, 'ts')).map((t) => t.type),
+ );
+ for (const t of ['keyword', 'string', 'comment', 'number', 'function', 'operator']) {
+ assert.ok(types.has(t), `expected a ${t} token in the sample`);
+ }
+ });
+
+ test('every prefix lexes without throwing, with valid invariants', () => {
+ for (let len = 0; len <= content.length; len += 13) {
+ const prefix = content.slice(0, len);
+ const lexed = syntax_styler_global.lex(prefix, 'ts');
+ const issues = validate_syntax_events(lexed);
+ assert.deepEqual(issues, [], `prefix of length ${len}`);
+ }
+ });
+
+ test('lexing is deterministic', () => {
+ const a = syntax_events_to_tokens(syntax_styler_global.lex(content, 'ts'));
+ const b = syntax_events_to_tokens(syntax_styler_global.lex(content, 'ts'));
+ assert.deepEqual(a, b);
+ });
+});
+
+describe('lexer_ts deep nesting', () => {
+ test('deeply nested template interpolations tokenize fully without overflowing the stack', () => {
+ const depth = 20000;
+ const input = '`${'.repeat(depth) + '1' + '}`'.repeat(depth);
+ const lexed = syntax_styler_global.lex(input, 'ts');
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ const types = syntax_events_to_tokens(lexed).map((t) => t.type);
+ assert.equal(types.filter((t) => t === 'template_string').length, depth);
+ assert.equal(types.filter((t) => t === 'interpolation').length, depth);
+ });
+
+ test('shallow nested templates tokenize their interiors', () => {
+ const types = tokens_of('`a${`b${c}d`}e`').map(([type]) => type);
+ assert.equal(types.filter((t) => t === 'template_string').length, 2);
+ assert.equal(types.filter((t) => t === 'interpolation').length, 2);
+ });
+});
+
+describe('lexer_ts as an embedded guest', () => {
+ // a host lexer embeds ts over `[0, split)` then emits its own token for the
+ // tail. Built-in embedders trim ts regions to a whitespace/delimiter
+ // boundary, but the public `Lexer.embed` API lets a custom lexer end the
+ // window mid-construct — where an unbounded number/spread peek used to read
+ // and emit one char past `split`, producing an out-of-window (invalid) span.
+ const T_TAIL = token_type('comment');
+ const embed_ts = (text: string, split: number) => {
+ const host: SyntaxLang = {
+ id: 'ts_embed_host',
+ lex: (l) => {
+ l.embed('ts', 0, split);
+ if (l.end > split) l.leaf(T_TAIL, split, l.end);
+ },
+ };
+ const styler = new SyntaxStyler();
+ styler.add_lang(lexer_ts);
+ styler.add_lang(host);
+ return styler.lex(text, 'ts_embed_host');
+ };
+
+ // each split lands mid-construct: after `e`/`e+`, inside `0x`, mid-`..`
+ for (const [text, split] of [
+ ['9e+5', 2],
+ ['9e-5', 2],
+ ['0xff', 1],
+ ['0n', 1],
+ ['a...', 3],
+ ['1.5', 2],
+ ] as Array<[string, number]>) {
+ test(`stays within its window: ${JSON.stringify(text)} @ ${split}`, () => {
+ const lexed = embed_ts(text, split);
+ // the guest must not emit past `split` — else it overlaps the host tail
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ for (const t of syntax_events_to_tokens(lexed)) {
+ assert.ok(
+ !(t.start < split && t.end > split),
+ `token ${t.type} [${t.start},${t.end}) crosses the embed boundary ${split}`,
+ );
+ }
+ });
+ }
+
+ test('a spread fully inside the window is still recognized', () => {
+ const lexed = embed_ts('...x', 4);
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ assert.ok(
+ syntax_events_to_tokens(lexed).some((t) => t.type === 'operator' && t.end - t.start === 3),
+ );
+ });
+});
diff --git a/src/test/pathological.ts b/src/test/pathological.ts
new file mode 100644
index 00000000..840a27ce
--- /dev/null
+++ b/src/test/pathological.ts
@@ -0,0 +1,250 @@
+/**
+ * Pathological input generators for the lexer engine — workload classes that
+ * realistic samples don't cover: deep nesting, huge single lines, very long
+ * and escape-heavy strings, many tiny tokens, colon/angle/paren-dense
+ * text that stresses the ts lexer's bounded heuristic scans
+ * (`scan_type_annotation`, `scan_balanced_angle`, `is_function_variable`),
+ * and probe-starved text — dense in a construct char while lacking a char
+ * its scanner probes for (`<` vs `{` vs `&`, link `]`/`)`) — that stresses
+ * the monotonic probe caches (`advance_probe`).
+ *
+ * Shared by the linearity suite (`lexer.pathological.test.ts`) and the
+ * benchmark runner (`benchmark/benchmarks.ts`) so the same workloads are both
+ * CI-enforced and perf-tracked.
+ *
+ * @module
+ */
+
+export interface PathologicalCase {
+ name: string;
+ lang: string;
+ /**
+ * Generates an input of approximately `size` chars (at least one repetition
+ * of the case's building block, so tiny sizes still produce valid input).
+ */
+ generate: (size: number) => string;
+}
+
+const repeat_to_size = (unit: string, size: number): string =>
+ unit.repeat(Math.max(1, Math.round(size / unit.length)));
+
+// fixed nesting depth for the repeated-block deep-nesting cases — constant
+// depth repeated to reach the target size, measuring input-length scaling at
+// realistic nesting. The *_full_depth cases scale nesting depth with size
+// instead: linear because substitution/interpolation interiors discover their
+// own closing delimiters during real tokenization (a per-level close-prescan
+// would make them O(depth²)). Unbounded-depth stack safety is covered by the
+// per-lexer `deep nesting` test suites.
+const NESTING_DEPTH = 32;
+
+export const PATHOLOGICAL_CASES: Array = [
+ {
+ // every `:` triggers a type-annotation lookahead that finds no `=` —
+ // the measured trigger from the design sheet (§10)
+ name: 'ts_colon_dense',
+ lang: 'ts',
+ generate: (size) => repeat_to_size('a:', size),
+ },
+ {
+ // every ident is followed by `<` that never balances, so each one pays
+ // the bounded generic-call angle scan
+ name: 'ts_angle_dense',
+ lang: 'ts',
+ generate: (size) => repeat_to_size('a<', size),
+ },
+ {
+ // every ident is followed by `= (` with no closing paren in reach, so
+ // `is_function_variable` pays the bounded balanced-paren scan
+ name: 'ts_paren_dense',
+ lang: 'ts',
+ generate: (size) => repeat_to_size('a=(', size),
+ },
+ {
+ // template literals with interpolations nested NESTING_DEPTH deep
+ name: 'ts_deep_template',
+ lang: 'ts',
+ generate: (size) =>
+ repeat_to_size('`${'.repeat(NESTING_DEPTH) + '1' + '}`'.repeat(NESTING_DEPTH) + ';\n', size),
+ },
+ {
+ // one enormous line of ordinary code — no newlines anywhere
+ name: 'ts_huge_line',
+ lang: 'ts',
+ generate: (size) => repeat_to_size("const x1 = fn(a, 'str') + 42; ", size),
+ },
+ {
+ // a single giant string literal
+ name: 'ts_long_string',
+ lang: 'ts',
+ generate: (size) => `const s = "${'x'.repeat(Math.max(1, size - 14))}";`,
+ },
+ {
+ // a single string that is nothing but escapes
+ name: 'ts_escape_heavy',
+ lang: 'ts',
+ generate: (size) => `const s = "${'\\x'.repeat(Math.max(1, Math.floor(size / 2) - 7))}";`,
+ },
+ {
+ // maximum token density — operator/number/punctuation every few chars
+ name: 'ts_many_tiny',
+ lang: 'ts',
+ generate: (size) => repeat_to_size('a=1;', size),
+ },
+ {
+ // one giant punctuation run — stresses emit-time span coalescing
+ name: 'json_deep_array',
+ lang: 'json',
+ generate: (size: number): string => {
+ const half = Math.max(1, Math.floor((size - 1) / 2));
+ return '['.repeat(half) + '1' + ']'.repeat(half);
+ },
+ },
+ {
+ // every `<` opens a tag that never closes — one giant unterminated tag
+ // churning through attribute names
+ name: 'html_angle_flood',
+ lang: 'html',
+ generate: (size) => repeat_to_size(' repeat_to_size('', size),
+ },
+ {
+ // entity-dense text
+ name: 'html_entity_dense',
+ lang: 'html',
+ generate: (size) => repeat_to_size('&x', size),
+ },
+ {
+ // expression-dense template text — balancer + ts embed per `{…}`
+ name: 'svelte_expression_dense',
+ lang: 'svelte',
+ generate: (size) => repeat_to_size('{x}', size),
+ },
+ {
+ // emphasis-delimiter-dense text — every `*`/`_` probes for a closer
+ name: 'md_emphasis_dense',
+ lang: 'md',
+ generate: (size) => repeat_to_size('*a _b ', size),
+ },
+ {
+ // fence-dense markdown — block scanner + close-fence line scans
+ name: 'md_fence_dense',
+ lang: 'md',
+ generate: (size) => repeat_to_size('```ts\nlet x = 1;\n```\n', size),
+ },
+ {
+ // `[`-dense text with no closing `]` — each `[` probes for a link closer
+ // that never comes; a naive per-`[` forward scan would be quadratic
+ name: 'md_bracket_dense',
+ lang: 'md',
+ generate: (size) => repeat_to_size('[', size),
+ },
+ {
+ // `[x](`-dense text — exercises both the `]` and `)` link-closer probes
+ // with no closing `)` in reach
+ name: 'md_link_paren_dense',
+ lang: 'md',
+ generate: (size) => repeat_to_size('[x](', size),
+ },
+ {
+ // expression-valued attributes — tag scanner + expression interplay
+ name: 'svelte_attr_expr_dense',
+ lang: 'svelte',
+ generate: (size) => repeat_to_size('', size),
+ },
+ {
+ // tag-dense svelte with no `{` anywhere — the expression-brace probe
+ // never hits, so it must be cached rather than re-run per tag
+ name: 'svelte_tag_only_dense',
+ lang: 'svelte',
+ generate: (size) => repeat_to_size('a', size),
+ },
+ {
+ // tag-dense html with no `&` anywhere — the entity probe never hits, so
+ // it must be cached across text gaps and attribute values
+ name: 'html_no_entity_dense',
+ lang: 'html',
+ generate: (size) => repeat_to_size('a', size),
+ },
+ {
+ // one raw-markup tag per line — md threads the markup probe caches
+ // across lines, so an absent `&` costs one probe per document, not one
+ // per line
+ name: 'md_tag_per_line',
+ lang: 'md',
+ generate: (size) => repeat_to_size('x\n', size),
+ },
+ {
+ // one link opener per line with no `)` anywhere — the link probes must
+ // stay monotonic across lines, not just within one
+ name: 'md_link_paren_per_line',
+ lang: 'md',
+ generate: (size) => repeat_to_size('[x](y\n', size),
+ },
+ {
+ // nested rules NESTING_DEPTH deep — the native-nesting state stack
+ name: 'css_deep_nesting',
+ lang: 'css',
+ generate: (size) =>
+ repeat_to_size('a{'.repeat(NESTING_DEPTH) + 'color:red;' + '}'.repeat(NESTING_DEPTH), size),
+ },
+ {
+ // command substitutions NESTING_DEPTH deep
+ name: 'bash_deep_command_sub',
+ lang: 'bash',
+ generate: (size) =>
+ repeat_to_size(
+ 'echo $('.repeat(NESTING_DEPTH) + 'ls' + ')'.repeat(NESTING_DEPTH) + '\n',
+ size,
+ ),
+ },
+ {
+ // one interpolation nest as deep as the input allows — every char is a
+ // delimiter of the same construct, so any per-level rescan is quadratic
+ name: 'ts_template_full_depth',
+ lang: 'ts',
+ generate: (size: number): string => {
+ const depth = Math.max(1, Math.floor(size / 5));
+ return '`${'.repeat(depth) + '1' + '}`'.repeat(depth);
+ },
+ },
+ {
+ // one command-substitution nest as deep as the input allows
+ name: 'bash_cmdsub_full_depth',
+ lang: 'bash',
+ generate: (size: number): string => {
+ const depth = Math.max(1, Math.floor(size / 3));
+ return '$('.repeat(depth) + 'x' + ')'.repeat(depth);
+ },
+ },
+ {
+ // heredocs opened inside command substitutions, nested as deep as the
+ // input allows — each body suspends at the next `$(`, so the closing
+ // delimiter must be discovered in the same forward pass (a per-heredoc
+ // close-prescan would be O(depth²), the bug this case guards against)
+ name: 'bash_heredoc_sub_full_depth',
+ lang: 'bash',
+ generate: (size: number): string => {
+ const depth = Math.max(1, Math.floor(size / 12));
+ return '$(cat < repeat_to_size('cat < repeat_to_size('```md\n', size),
+ },
+];
diff --git a/src/test/svelte_preprocess_fuz_code.test.ts b/src/test/svelte_preprocess_fuz_code.test.ts
index f33df61e..fd8da448 100644
--- a/src/test/svelte_preprocess_fuz_code.test.ts
+++ b/src/test/svelte_preprocess_fuz_code.test.ts
@@ -503,22 +503,21 @@ const y = 2;" lang="ts" />`;
const raw_html = extract_raw_html(result);
assert.include(raw_html!, '<');
});
- });
- describe('skip conditions', () => {
- test('skips custom grammar', async () => {
+ test('skips static content that stylizes to itself', async () => {
const input = `
-{x}
';
- const result = syntax_styler_global.stylize(code, 'svelte');
- assert.ok(result.includes('tag'), 'Should recognize HTML tags');
- // Note: embedded language support depends on implementation
- });
-});
-
-describe('lastIndex and position management', () => {
- test('lastIndex properly reset for greedy patterns', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Test with code that has multiple strings at different positions
- const code1 = ' "first" "second"';
- const code2 = '"immediate" "next"';
-
- const result1 = syntax_styler.stylize(code1, 'js');
- const result2 = syntax_styler.stylize(code2, 'js');
-
- // Both should tokenize correctly despite different starting positions
- assert.ok(result1.includes('"first"'));
- assert.ok(result1.includes('"second"'));
- assert.ok(result2.includes('"immediate"'));
- assert.ok(result2.includes('"next"'));
- });
-
- test('greedy patterns work with strings at various positions', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Strings at different positions in the text
- const code = `
- "start"
- "indented"
- const x = "inline";
- "far right"
- `;
-
- const result = syntax_styler.stylize(code, 'js');
-
- // All strings should be tokenized regardless of position
- const string_count = (result.match(//g) || []).length;
- assert.equal(string_count, 4, 'Should tokenize all 4 strings');
- });
-});
-
-describe('pattern flag edge cases', () => {
- test('patterns with existing global flag not double-processed', () => {
- const syntax_styler = new SyntaxStyler();
-
- // Store the original patterns BEFORE registration
- const patterns_before = {
- already_global: /test/g, // Already has g flag
- case_insensitive: /test/i, // Has i flag but not g
- multi_flag: /test/gim, // Multiple flags including g
- };
-
- // Add a test grammar with patterns that already have flags
- syntax_styler.add_lang('test', {
- already_global: {
- pattern: patterns_before.already_global,
- greedy: true,
- },
- case_insensitive: {
- pattern: patterns_before.case_insensitive,
- greedy: true,
- },
- multi_flag: {
- pattern: patterns_before.multi_flag,
- greedy: true,
- },
+import {syntax_events_to_tokens, validate_syntax_events, type SyntaxLang} from '$lib/lexer.ts';
+
+describe('SyntaxStyler registry', () => {
+ test('has_lang reflects registration and aliases', () => {
+ const styler = new SyntaxStyler();
+ assert.ok(styler.has_lang('plaintext')); // registered by default
+ assert.ok(!styler.has_lang('ts'));
+ const lang: SyntaxLang = {id: 'foo', aliases: ['bar'], lex: (l) => (l.pos = l.end)};
+ styler.add_lang(lang);
+ assert.ok(styler.has_lang('foo'));
+ assert.ok(styler.has_lang('bar'));
+ assert.ok(!styler.has_lang('baz'));
+ });
+
+ test('lex throws on an unregistered language', () => {
+ const styler = new SyntaxStyler();
+ assert.throws(() => styler.lex('x', 'nope'), /not registered/);
+ assert.throws(() => styler.stylize('x', 'nope'), /not registered/);
+ });
+
+ test('plaintext emits no tokens and renders escaped but unstyled', () => {
+ const lexed = syntax_styler_global.lex('a < b & c', 'plaintext');
+ assert.equal(syntax_events_to_tokens(lexed).length, 0);
+ assert.equal(syntax_styler_global.stylize('a < b & c', 'plaintext'), 'a < b & c');
+ });
+});
+
+describe('syntax_styler_global aliases', () => {
+ // each alias must resolve to a registered lexer and produce a valid,
+ // non-empty event stream — guards against a lexer's aliases going unwired
+ const cases: Array<[string, string]> = [
+ ['js', 'const x = 1;'],
+ ['javascript', 'const x = 1;'],
+ ['typescript', 'const x = 1;'],
+ ['sh', 'echo hi'],
+ ['shell', 'echo hi'],
+ ['svg', ''],
+ ['mathml', ''],
+ ['xml', ''],
+ ['ssml', ''],
+ ['atom', ''],
+ ['rss', ''],
+ ];
+ for (const [alias, src] of cases) {
+ test(`alias "${alias}" is registered and lexes`, () => {
+ assert.ok(syntax_styler_global.has_lang(alias), `${alias} should be registered`);
+ const lexed = syntax_styler_global.lex(src, alias);
+ assert.deepEqual(validate_syntax_events(lexed), []);
+ assert.ok(syntax_events_to_tokens(lexed).length > 0, `${alias} should emit tokens`);
});
-
- const grammar = syntax_styler.get_lang('test');
- // After normalization, grammar values are arrays and patterns have been normalized
-
- // Patterns that already had g flag should keep the same RegExp instance
- assert.equal(
- grammar.already_global![0]!.pattern,
- patterns_before.already_global,
- 'Already global pattern should not be replaced',
- );
- assert.equal(
- grammar.multi_flag![0]!.pattern,
- patterns_before.multi_flag,
- 'Multi-flag pattern should not be replaced',
- );
-
- // Pattern without g gets a new RegExp with g added during normalization
- assert.notEqual(
- grammar.case_insensitive![0]!.pattern,
- patterns_before.case_insensitive,
- 'Pattern without g gets replaced with global version during normalization',
- );
- assert.equal(
- grammar.case_insensitive![0]!.pattern.flags,
- 'gi', // Now has both flags
- 'Pattern gains global flag',
- );
-
- // After tokenization, patterns remain unchanged (no runtime mutation)
- syntax_styler.stylize('test TEST tEsT', 'test');
-
- assert.equal(
- grammar.already_global![0]!.pattern,
- patterns_before.already_global,
- 'Pattern unchanged after tokenization',
- );
- });
-
- test('non-greedy patterns remain untouched', () => {
- const syntax_styler = create_styler_with_grammars();
- const grammar = syntax_styler.get_lang('js');
-
- // Get non-greedy patterns (keyword is an array of patterns)
- const keyword_patterns = grammar.keyword as Array;
- const original_first = keyword_patterns[0];
- const original_second = keyword_patterns[1];
-
- syntax_styler.stylize('const let var function class extends', 'js');
-
- // Non-greedy patterns should remain exactly the same
- assert.equal(keyword_patterns[0], original_first, 'First keyword pattern unchanged');
- assert.equal(keyword_patterns[1], original_second, 'Second keyword pattern unchanged');
- });
-});
-
-describe('multiple SyntaxStyler instances', () => {
- test('separate instances have independent grammars', () => {
- const styler1 = new SyntaxStyler();
- const styler2 = new SyntaxStyler();
-
- // Load grammars into both
- add_grammar_markup(styler1);
- add_grammar_clike(styler1);
- add_grammar_js(styler1);
-
- add_grammar_markup(styler2);
- add_grammar_clike(styler2);
- add_grammar_js(styler2);
-
- const grammar1 = styler1.get_lang('js');
- const grammar2 = styler2.get_lang('js');
-
- // Store patterns after normalization (grammar values are arrays)
- const pattern1 = grammar1.string![0]!.pattern;
- const pattern2 = grammar2.string![0]!.pattern;
-
- // Patterns should be different objects (each instance has its own)
- assert.notEqual(pattern1, pattern2, 'Each instance has separate pattern objects');
-
- // Verify both patterns already have the global flag from normalization
- assert.ok(pattern1.flags.includes('g'), 'styler1 pattern has g flag from normalization');
- assert.ok(pattern2.flags.includes('g'), 'styler2 pattern has g flag from normalization');
-
- // Tokenizing with one shouldn't affect the other
- styler1.stylize('"test"', 'js');
- styler2.stylize('"test"', 'js');
-
- // Patterns remain unchanged after tokenization (no runtime mutation)
- assert.equal(
- grammar1.string![0]!.pattern,
- pattern1,
- 'styler1 pattern unchanged after tokenization',
- );
- assert.equal(
- grammar2.string![0]!.pattern,
- pattern2,
- 'styler2 pattern unchanged after tokenization',
- );
-
- // Each instance still has its own pattern object
- assert.notEqual(
- grammar1.string![0]!.pattern,
- grammar2.string![0]!.pattern,
- 'Instances remain independent',
- );
- });
-});
-
-describe('nested tokenization', () => {
- test('template strings with nested interpolation', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Deeply nested template strings
- const code = '`outer ${`inner ${`deepest ${x}`} middle`} end`';
- const result = syntax_styler.stylize(code, 'js');
-
- // Should handle all levels of nesting
- assert.ok(result.includes('template'), 'Should recognize template strings');
- assert.ok(result.includes('interpolation'), 'Should recognize interpolations');
-
- // Count interpolations (should be 3)
- const interpolation_count = (result.match(/interpolation/g) || []).length;
- assert.ok(interpolation_count >= 3, 'Should handle all nested interpolations');
- });
-
- test('regex pattern with special characters', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Complex regex that might break if caching is wrong
- const code = 'const r = /\\$\\{[^}]+\\}/g;'; // Regex that looks like template syntax
- const result = syntax_styler.stylize(code, 'js');
-
- assert.ok(result.includes('regex'), 'Should recognize as regex');
- assert.ok(!result.includes('template'), 'Should not confuse with template string');
- });
-});
-
-describe('tokenization consistency', () => {
- test('same pattern used multiple times in one tokenization', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Many strings to ensure pattern is reused
- const code = '"a" + "b" + "c" + "d" + "e" + "f" + "g" + "h"';
- const result = syntax_styler.stylize(code, 'js');
-
- // All strings should be tokenized
- const string_count = (result.match(//g) || []).length;
- assert.equal(string_count, 8, 'Should tokenize all 8 strings');
- });
-
- test('empty strings handled correctly', () => {
- const syntax_styler = create_styler_with_grammars();
-
- const code = 'const empty = "";';
- const result = syntax_styler.stylize(code, 'js');
-
- assert.ok(result.includes('string'), 'Should tokenize empty string');
- assert.ok(result.includes('""'), 'Should preserve empty string content');
- });
-
- test('very long strings handled correctly', () => {
- const syntax_styler = create_styler_with_grammars();
-
- // Create a very long string to test lastIndex with large values
- const long_content = 'x'.repeat(10000);
- const code = `const long = "${long_content}";`;
- const result = syntax_styler.stylize(code, 'js');
-
- assert.ok(result.includes('string'), 'Should tokenize long string');
- assert.ok(result.includes(long_content), 'Should preserve long string content');
- });
-});
-
-describe('tokenize (hooked tokenization)', () => {
- test('matches the bare tokenize_syntax stream when no hooks are registered', () => {
- const syntax_styler = create_styler_with_grammars();
- const code = 'const x = 1;';
- assert.deepEqual(
- syntax_styler.tokenize(code, 'js'),
- tokenize_syntax(code, syntax_styler.get_lang('js')),
- );
- });
-
- test('runs before_tokenize and after_tokenize hooks (which bare tokenize_syntax skips)', () => {
- const syntax_styler = create_styler_with_grammars();
- let before_lang: string | undefined;
- let after_token_count: number | undefined;
- syntax_styler.add_hook_before_tokenize((ctx) => {
- before_lang = ctx.lang;
- });
- syntax_styler.add_hook_after_tokenize((ctx) => {
- after_token_count = ctx.tokens.length;
- });
-
- const tokens = syntax_styler.tokenize('const x = 1;', 'js');
-
- assert.equal(before_lang, 'js');
- assert.equal(after_token_count, tokens.length);
- });
-
- test('a before_tokenize hook can rewrite the code before tokenization', () => {
- const syntax_styler = create_styler_with_grammars();
- syntax_styler.add_hook_before_tokenize((ctx) => {
- ctx.code = ctx.code.replace('var', 'const');
- });
-
- const tokens = syntax_styler.tokenize('var x = 1;', 'js');
- const keyword = tokens.find((t) => typeof t !== 'string' && t.type === 'keyword');
- assert.ok(keyword && typeof keyword !== 'string');
- assert.equal(keyword.content, 'const');
- });
-
- test('an after_tokenize hook can replace the token stream', () => {
- const syntax_styler = create_styler_with_grammars();
- syntax_styler.add_hook_after_tokenize((ctx) => {
- ctx.tokens = ['replaced'];
- });
- assert.deepEqual(syntax_styler.tokenize('const x = 1;', 'js'), ['replaced']);
- });
-
- test('stylize stringifies with the lang a before_tokenize hook rewrote', () => {
- const syntax_styler = create_styler_with_grammars();
- let wrap_lang: string | undefined;
- // the rewritten lang must flow through to each token's wrap hook context
- syntax_styler.add_hook_before_tokenize((ctx) => {
- ctx.lang = 'rewritten';
- });
- syntax_styler.add_hook_wrap((ctx) => {
- wrap_lang = ctx.lang;
- });
-
- syntax_styler.stylize('const x = 1;', 'js');
- assert.equal(wrap_lang, 'rewritten');
- });
+ }
});