From 94cbe1b742355fa25c5b87c0b1a74adfc59c5fe2 Mon Sep 17 00:00:00 2001 From: Joey Holiga Date: Fri, 14 Aug 2026 15:34:48 -0400 Subject: [PATCH] Replace scan extraction with @dittowords/text-extract The CLI held its own copy of the extraction pipeline. That code now ships as @dittowords/text-extract. Import the library and delete the local copy. Changes: - Add @dittowords/text-extract as a dependency. - Delete lib/src/scan/{extract,walk,rules,types}.ts and lib/src/scan/lang. - Delete the tests for the deleted code. The library tests it. - Import DittoScanCandidate, DittoScanExtractSummary and runExtract from the library. - Re-export the scan types from the library in lib/ditto.ts. - Log the new summary.filesFailed count and the failure detail. A failed file drops out of the results, so its strings are missing. - Remove @ast-grep/napi, @ast-grep/lang-kotlin, @ast-grep/lang-swift, globby and yaml. The library owns them now. - Add lib/src/scan/extract.test.ts. It checks that the CLI resolves the library and gets usable candidates from the test fixtures. lib/src/scan/analyzeDirectories.ts stays. It is CLI-only code. The output does not change. A scan of testfiles/ and of lib/ gives candidates that are identical to the candidates from master. Co-Authored-By: Claude Opus 5 (1M context) --- jest.config.ts | 5 +- lib/ditto.ts | 2 +- lib/src/commands/scan.ts | 14 +- lib/src/http/scan.ts | 2 +- lib/src/scan/extract.test.ts | 28 ++ lib/src/scan/extract.ts | 311 ------------- .../lang/extractors/android-resources.test.ts | 133 ------ .../scan/lang/extractors/android-resources.ts | 112 ----- lib/src/scan/lang/extractors/arb.ts | 23 - lib/src/scan/lang/extractors/fallback.test.ts | 41 -- lib/src/scan/lang/extractors/fallback.ts | 43 -- .../scan/lang/extractors/html-markup.test.ts | 50 --- lib/src/scan/lang/extractors/html-markup.ts | 181 -------- .../scan/lang/extractors/javascript.test.ts | 107 ----- lib/src/scan/lang/extractors/javascript.ts | 409 ------------------ .../scan/lang/extractors/json-i18n.test.ts | 76 ---- lib/src/scan/lang/extractors/json-i18n.ts | 273 ------------ lib/src/scan/lang/extractors/kotlin.test.ts | 73 ---- lib/src/scan/lang/extractors/kotlin.ts | 219 ---------- lib/src/scan/lang/extractors/po.test.ts | 69 --- lib/src/scan/lang/extractors/po.ts | 211 --------- lib/src/scan/lang/extractors/properties.ts | 94 ---- lib/src/scan/lang/extractors/resx.test.ts | 16 - lib/src/scan/lang/extractors/resx.ts | 85 ---- lib/src/scan/lang/extractors/strings.test.ts | 55 --- lib/src/scan/lang/extractors/strings.ts | 129 ------ .../scan/lang/extractors/stringsdict.test.ts | 55 --- lib/src/scan/lang/extractors/stringsdict.ts | 108 ----- lib/src/scan/lang/extractors/swift.test.ts | 68 --- lib/src/scan/lang/extractors/swift.ts | 214 --------- lib/src/scan/lang/extractors/util.test.ts | 32 -- lib/src/scan/lang/extractors/util.ts | 32 -- lib/src/scan/lang/extractors/vue.test.ts | 47 -- lib/src/scan/lang/extractors/vue.ts | 89 ---- .../scan/lang/extractors/xcstrings.test.ts | 75 ---- lib/src/scan/lang/extractors/xcstrings.ts | 308 ------------- lib/src/scan/lang/extractors/xliff.ts | 51 --- lib/src/scan/lang/extractors/xml.ts | 133 ------ .../scan/lang/extractors/yaml-i18n.test.ts | 45 -- lib/src/scan/lang/extractors/yaml-i18n.ts | 118 ----- lib/src/scan/lang/file-discovery.ts | 146 ------- lib/src/scan/lang/i18n-file-discovery.test.ts | 95 ---- lib/src/scan/lang/i18n-file-discovery.ts | 283 ------------ lib/src/scan/lang/registry.ts | 140 ------ lib/src/scan/lang/types.ts | 22 - lib/src/scan/rules.test.ts | 106 ----- lib/src/scan/rules.ts | 190 -------- lib/src/scan/types.ts | 109 ----- lib/src/scan/walk.ts | 210 --------- package.json | 6 +- yarn.lock | 11 + 51 files changed, 57 insertions(+), 5397 deletions(-) create mode 100644 lib/src/scan/extract.test.ts delete mode 100644 lib/src/scan/extract.ts delete mode 100644 lib/src/scan/lang/extractors/android-resources.test.ts delete mode 100644 lib/src/scan/lang/extractors/android-resources.ts delete mode 100644 lib/src/scan/lang/extractors/arb.ts delete mode 100644 lib/src/scan/lang/extractors/fallback.test.ts delete mode 100644 lib/src/scan/lang/extractors/fallback.ts delete mode 100644 lib/src/scan/lang/extractors/html-markup.test.ts delete mode 100644 lib/src/scan/lang/extractors/html-markup.ts delete mode 100644 lib/src/scan/lang/extractors/javascript.test.ts delete mode 100644 lib/src/scan/lang/extractors/javascript.ts delete mode 100644 lib/src/scan/lang/extractors/json-i18n.test.ts delete mode 100644 lib/src/scan/lang/extractors/json-i18n.ts delete mode 100644 lib/src/scan/lang/extractors/kotlin.test.ts delete mode 100644 lib/src/scan/lang/extractors/kotlin.ts delete mode 100644 lib/src/scan/lang/extractors/po.test.ts delete mode 100644 lib/src/scan/lang/extractors/po.ts delete mode 100644 lib/src/scan/lang/extractors/properties.ts delete mode 100644 lib/src/scan/lang/extractors/resx.test.ts delete mode 100644 lib/src/scan/lang/extractors/resx.ts delete mode 100644 lib/src/scan/lang/extractors/strings.test.ts delete mode 100644 lib/src/scan/lang/extractors/strings.ts delete mode 100644 lib/src/scan/lang/extractors/stringsdict.test.ts delete mode 100644 lib/src/scan/lang/extractors/stringsdict.ts delete mode 100644 lib/src/scan/lang/extractors/swift.test.ts delete mode 100644 lib/src/scan/lang/extractors/swift.ts delete mode 100644 lib/src/scan/lang/extractors/util.test.ts delete mode 100644 lib/src/scan/lang/extractors/util.ts delete mode 100644 lib/src/scan/lang/extractors/vue.test.ts delete mode 100644 lib/src/scan/lang/extractors/vue.ts delete mode 100644 lib/src/scan/lang/extractors/xcstrings.test.ts delete mode 100644 lib/src/scan/lang/extractors/xcstrings.ts delete mode 100644 lib/src/scan/lang/extractors/xliff.ts delete mode 100644 lib/src/scan/lang/extractors/xml.ts delete mode 100644 lib/src/scan/lang/extractors/yaml-i18n.test.ts delete mode 100644 lib/src/scan/lang/extractors/yaml-i18n.ts delete mode 100644 lib/src/scan/lang/file-discovery.ts delete mode 100644 lib/src/scan/lang/i18n-file-discovery.test.ts delete mode 100644 lib/src/scan/lang/i18n-file-discovery.ts delete mode 100644 lib/src/scan/lang/registry.ts delete mode 100644 lib/src/scan/lang/types.ts delete mode 100644 lib/src/scan/rules.test.ts delete mode 100644 lib/src/scan/rules.ts delete mode 100644 lib/src/scan/types.ts delete mode 100644 lib/src/scan/walk.ts diff --git a/jest.config.ts b/jest.config.ts index 0c16749..0be2911 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -2,9 +2,10 @@ import type { Config } from "jest"; const config: Config = { transformIgnorePatterns: [], + // globby v16 is ESM-only. Jest resolves its `unicorn-magic/node` subpath + // import to the ESM entry, which babel-jest cannot load. moduleNameMapper: { - "^unicorn-magic/node$": - "/node_modules/unicorn-magic/node.js", + "^unicorn-magic/node$": "/node_modules/unicorn-magic/node.js", }, maxWorkers: 1, verbose: true, diff --git a/lib/ditto.ts b/lib/ditto.ts index bdf04eb..0e06432 100755 --- a/lib/ditto.ts +++ b/lib/ditto.ts @@ -25,6 +25,6 @@ const main = async () => { } }; -export type * from "./src/scan/types"; +export type * from "@dittowords/text-extract"; main(); diff --git a/lib/src/commands/scan.ts b/lib/src/commands/scan.ts index a5ecc0b..a8dd935 100644 --- a/lib/src/commands/scan.ts +++ b/lib/src/commands/scan.ts @@ -5,8 +5,11 @@ import { prompt } from "enquirer"; import open from "open"; import logger from "../utils/logger"; -import { DittoScanExtractSummary, runExtract } from "../scan/extract"; -import { DittoScanCandidate } from "../scan/types"; +import { + DittoScanCandidate, + DittoScanExtractSummary, + runExtract, +} from "@dittowords/text-extract"; import { quit } from "../utils/quit"; import initAPIToken from "../services/apiToken/initAPIToken"; import appContext from "../utils/appContext"; @@ -91,6 +94,13 @@ function logExtractSummary( ` files.skipped_minified: ${summary.filesSkippedMinified}\n` ); } + // A failed file is dropped from the results, so its strings are missing. + if (summary.filesFailed > 0) { + process.stderr.write(` files.failed: ${summary.filesFailed}\n`); + for (const f of summary.failures) { + process.stderr.write(` ${f.file} (${f.language}): ${f.message}\n`); + } + } process.stderr.write( `[ditto-cli scan][extract] emitted ${ summary.candidatesEmitted diff --git a/lib/src/http/scan.ts b/lib/src/http/scan.ts index acaaf17..87338d6 100644 --- a/lib/src/http/scan.ts +++ b/lib/src/http/scan.ts @@ -1,7 +1,7 @@ import axios, { AxiosError } from "axios"; import getHttpClient from "./client"; import { IInitiateScanResponse, ZInitiateScanResponse } from "./types"; -import { DittoScanCandidate } from "../scan/types"; +import { DittoScanCandidate } from "@dittowords/text-extract"; import DittoError, { ErrorType } from "../utils/DittoError"; import { Blob } from "buffer"; diff --git a/lib/src/scan/extract.test.ts b/lib/src/scan/extract.test.ts new file mode 100644 index 0000000..ccdd9d9 --- /dev/null +++ b/lib/src/scan/extract.test.ts @@ -0,0 +1,28 @@ +import path from "path"; + +import { runExtract } from "@dittowords/text-extract"; + +// The extraction itself is the library's to test. This only checks that the +// CLI resolves the package and gets usable candidates back from a real +// directory, so a bad dependency or a changed entry point fails here. +describe("runExtract", () => { + const testfiles = path.resolve(__dirname, "../../../testfiles"); + + it("extracts candidates from the test fixtures", async () => { + const { candidates, summary } = await runExtract({ inputPath: testfiles }); + + expect(summary.filesFailed).toBe(0); + expect(summary.filesScanned).toBeGreaterThan(0); + expect(candidates.length).toBe(summary.candidatesEmitted); + + const values = candidates.map((c) => c.value_raw); + expect(values).toContain("Hello World!"); + + // Every candidate carries the fields the CLI reads downstream. + for (const c of candidates) { + expect(typeof c.id).toBe("string"); + expect(c.location.file).not.toBe(""); + expect(path.isAbsolute(c.location.file)).toBe(false); + } + }); +}); diff --git a/lib/src/scan/extract.ts b/lib/src/scan/extract.ts deleted file mode 100644 index fdde4f0..0000000 --- a/lib/src/scan/extract.ts +++ /dev/null @@ -1,311 +0,0 @@ -import fs from "fs/promises"; -import { globby } from "globby"; -import path from "path"; - -import { - DittoScanDetectionKindSchema, - type DittoScanCandidate, - type DittoScanDetectionKind, -} from "./types"; -import { createHash } from "crypto"; -import type { FileDiscoveryStats } from "./lang/file-discovery"; -import { shouldEmit } from "./rules"; -import { walkCodebase } from "./walk"; - -export interface DittoScanExtractOptions { - inputPath: string; -} - -export interface DittoScanExtractResult { - candidates: DittoScanCandidate[]; - summary: DittoScanExtractSummary; -} - -export interface DittoScanExtractSummary { - filesScanned: number; - filesByKind: Record; - filesSkippedMinified: number; - candidatesEmitted: number; - candidatesByKind: Record; - framework: string[]; - elapsedMs: number; - i18nFileDiscovery: FileDiscoveryStats | null; -} - -const CONTEXT_LINES = 3; -const MAX_CONTEXT_LINE_CHARS = 200; - -// Maps a dependency name in package.json to the framework token we surface -// to the LLM. Only frameworks that meaningfully shift the user-facing -// likelihood of strings are listed (UI frameworks, server frameworks). -const FRAMEWORK_MARKERS: ReadonlyArray< - readonly [pattern: RegExp, token: string] -> = [ - [/^react-native$/, "react-native"], - [/^react(-dom)?$/, "react"], - [/^next$/, "next"], - [/^@remix-run\//, "remix"], - [/^vue$/, "vue"], - [/^nuxt$/, "nuxt"], - [/^svelte$/, "svelte"], - [/^@sveltejs\/kit$/, "sveltekit"], - [/^@angular\/core$/, "angular"], - [/^solid-js$/, "solid"], - [/^preact$/, "preact"], - [/^astro$/, "astro"], - [/^express$/, "express"], - [/^fastify$/, "fastify"], - [/^koa$/, "koa"], - [/^@nestjs\/core$/, "nestjs"], - [/^@hapi\/hapi$/, "hapi"], -]; - -function zeroKindCounts(): Record { - return Object.fromEntries( - DittoScanDetectionKindSchema.options.map((k) => [k, 0]) - ) as Record; -} - -function buildSourceContext(lines: string[], targetLine: number): string { - const start = Math.max(1, targetLine - CONTEXT_LINES); - const end = Math.min(lines.length, targetLine + CONTEXT_LINES); - const out: string[] = []; - for (let i = start; i <= end; i++) { - const raw = lines[i - 1] ?? ""; - const text = - raw.length > MAX_CONTEXT_LINE_CHARS - ? raw.slice(0, MAX_CONTEXT_LINE_CHARS) + "…(truncated)" - : raw; - out.push(`${i}: ${text}`); - } - return out.join("\n"); -} - -const VCS_MARKERS = [".git", ".hg", ".svn"] as const; - -// Aggregates deps from every package.json: -// - Walking *up* from inputPath to the repo root, since monorepo -// subpackages often have a near-empty package.json with the real -// deps living one or more levels up. -// - Walking *down* from inputPath via gitignore-aware globby, since -// the inverse is also common: pnpm/yarn workspace monorepos where -// the root package.json is empty and react/vue/etc. live in -// `packages/*/package.json`. Without this pass, large frontend -// monorepos (excalidraw, nx-style repos) would surface -// `framework: (none detected)`. -// -// On top of the package.json passes, we sniff for native Android/iOS -// project markers anywhere under inputPath so a Gradle-only or -// Xcode-only project still surfaces an `android` / `ios` token to the -// LLM. We never read the marker contents — presence alone is the signal. -async function detectFramework(inputPath: string): Promise { - const tokens = new Set(); - - const ingestPackageJson = (raw: string) => { - let pkg: { - dependencies?: Record; - devDependencies?: Record; - }; - try { - pkg = JSON.parse(raw); - } catch { - return; - } - for (const name of Object.keys({ - ...(pkg.dependencies ?? {}), - ...(pkg.devDependencies ?? {}), - })) { - for (const [pattern, token] of FRAMEWORK_MARKERS) { - if (pattern.test(name)) tokens.add(token); - } - } - }; - - // Upward walk (inputPath → repo root). - let dir = path.resolve(inputPath); - while (true) { - try { - ingestPackageJson( - await fs.readFile(path.join(dir, "package.json"), "utf8") - ); - } catch { - // no package.json here, keep walking - } - if (await isRepoRoot(dir)) break; - const parent = path.dirname(dir); - if (parent === dir) break; - dir = parent; - } - - // Downward walk (inputPath/**/package.json). Honors .gitignore and - // skips the usual large-directory denylist so we don't spelunk through - // node_modules / build artifacts. - const nested = await globby(["**/package.json"], { - cwd: path.resolve(inputPath), - gitignore: true, - onlyFiles: true, - ignore: [ - "**/node_modules/**", - "**/.git/**", - "**/build/**", - "**/dist/**", - "**/.next/**", - "**/.nuxt/**", - ], - followSymbolicLinks: false, - suppressErrors: true, - absolute: true, - }); - for (const pkgPath of nested) { - try { - ingestPackageJson(await fs.readFile(pkgPath, "utf8")); - } catch { - // unreadable / malformed, ignore - } - } - - for (const platformToken of await detectMobilePlatforms(inputPath)) { - tokens.add(platformToken); - } - - return [...tokens].sort(); -} - -const ANDROID_MARKER_RE = - /(?:^|\/)(?:AndroidManifest\.xml|build\.gradle(?:\.kts)?|settings\.gradle(?:\.kts)?)$/; -const IOS_MARKER_RE = - /(?:^|\/)(?:[^/]+\.xcodeproj\/project\.pbxproj|Package\.swift|Podfile)$/; - -async function detectMobilePlatforms(inputPath: string): Promise { - const tokens = new Set(); - const matches = await globby( - [ - "**/AndroidManifest.xml", - "**/build.gradle", - "**/build.gradle.kts", - "**/settings.gradle", - "**/settings.gradle.kts", - "**/*.xcodeproj/project.pbxproj", - "**/Package.swift", - "**/Podfile", - ], - { - cwd: path.resolve(inputPath), - gitignore: true, - onlyFiles: true, - ignore: ["**/node_modules/**", "**/.git/**", "**/build/**", "**/Pods/**"], - followSymbolicLinks: false, - suppressErrors: true, - } - ); - for (const m of matches) { - if (ANDROID_MARKER_RE.test(m)) tokens.add("android"); - if (IOS_MARKER_RE.test(m)) tokens.add("ios"); - } - return [...tokens]; -} - -async function isRepoRoot(dir: string): Promise { - for (const marker of VCS_MARKERS) { - try { - await fs.access(path.join(dir, marker)); - return true; - } catch { - // marker not present, try the next one - } - } - return false; -} - -// Deterministic id for a candidate so the same string in the same place gets -// the same id across runs. -export function makeCandidateId( - file: string, - line: number, - column: number, - value: string -): string { - return createHash("sha1") - .update(`${file}:${line}:${column}:${value}`) - .digest("hex") - .slice(0, 12); -} - -export async function runExtract( - opts: DittoScanExtractOptions -): Promise { - const t0 = Date.now(); - const framework = await detectFramework(opts.inputPath); - const { files, filesSkippedMinified, i18nFileDiscovery } = await walkCodebase( - opts.inputPath - ); - - const filesByKind: Record = {}; - for (const f of files) - filesByKind[f.language.id] = (filesByKind[f.language.id] ?? 0) + 1; - - const candidatesByKind = zeroKindCounts(); - const candidates: DittoScanCandidate[] = []; - - for (const file of files) { - const lang = file.language; - let hits; - try { - hits = await lang.extractor.extract({ - source: file.source, - kind: lang.id, - }); - } catch (e) { - process.stderr.write( - `[ptd extract] failed on ${file.relPath} (${lang.id}): ${ - (e as Error).message - }\n` - ); - continue; - } - - const lines = file.source.split(/\r?\n/); - - for (const hit of hits) { - if (!shouldEmit(hit.value, hit.context)) continue; - const candidate: DittoScanCandidate = { - id: makeCandidateId( - file.relPath, - hit.location.line, - hit.location.column, - hit.value - ), - value_raw: hit.value, - detection_kind: hit.context.parentRole, - location: { - file: file.relPath, - line: hit.location.line, - column: hit.location.column, - }, - language: file.languageLabel, - locale_key: hit.localeKey ?? file.localeKey, - i18n_key: hit.i18nKey ?? null, - framework, - source_context: buildSourceContext(lines, hit.location.line), - context_identifiers: hit.context.identifiers, - usage_evidence: null, - }; - candidates.push(candidate); - candidatesByKind[hit.context.parentRole]++; - } - } - - return { - candidates, - summary: { - filesScanned: files.length, - filesByKind, - filesSkippedMinified, - candidatesEmitted: candidates.length, - candidatesByKind, - framework, - elapsedMs: Date.now() - t0, - i18nFileDiscovery, - }, - }; -} diff --git a/lib/src/scan/lang/extractors/android-resources.test.ts b/lib/src/scan/lang/extractors/android-resources.test.ts deleted file mode 100644 index 86dc7e1..0000000 --- a/lib/src/scan/lang/extractors/android-resources.test.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { androidResourceExtractor } from "./android-resources"; - -const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" }); - -describe("androidResourceExtractor", () => { - test("emits values keyed by name", async () => { - const source = [ - ``, - ``, - ` Hello, world!`, - ` Goodbye`, - ``, - ``, - ].join("\n"); - const hits = await extract(source); - expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([ - { v: "Hello, world!", ids: ["hello"] }, - { v: "Goodbye", ids: ["goodbye"] }, - ]); - expect(hits[0].context.parentRole).toBe("resource_value"); - }); - - test("emits one hit per item, tagged with the quantity", async () => { - const source = [ - ``, - ` `, - ` %d item`, - ` %d items`, - ` `, - ``, - ``, - ].join("\n"); - const hits = await extract(source); - expect(hits).toHaveLength(2); - expect(hits[0].context.identifiers).toEqual(["items", "one"]); - expect(hits[1].context.identifiers).toEqual(["items", "other"]); - }); - - test("emits one hit per item, indexed numerically", async () => { - const source = [ - ``, - ` `, - ` Mercury`, - ` Venus`, - ` `, - ``, - ``, - ].join("\n"); - const hits = await extract(source); - expect(hits.map((h) => h.value)).toEqual(["Mercury", "Venus"]); - expect(hits[0].context.identifiers).toEqual(["planets", "0"]); - expect(hits[1].context.identifiers).toEqual(["planets", "1"]); - }); - - test("recovers CDATA-wrapped values via the regex sweep", async () => { - const source = [ - ``, - ` Welcome]]>`, - ``, - ``, - ].join("\n"); - const hits = await extract(source); - const welcome = hits.find((h) => h.value === "Welcome"); - expect(welcome).toBeDefined(); - expect(welcome?.context.identifiers).toEqual(["welcome"]); - }); -}); - -// The main path. An earlier version of this fix only reached the CDATA -// sweep and silently did nothing here. -describe("androidResourceExtractor escape decoding", () => { - const extract = (source: string) => - androidResourceExtractor.extract({ source, kind: "android_resources" }); - - it("decodes escapes in a ", async () => { - const hits = await extract( - `line one\\nline two` - ); - expect(hits.map((h) => h.value)).toEqual(["line one\nline two"]); - }); - - it("decodes escapes inside items", async () => { - const hits = await extract( - ` - - one\\nfile - many\\nfiles - - ` - ); - expect(hits.map((h) => h.value)).toEqual(["one\nfile", "many\nfiles"]); - }); -}); - -// Ditto wraps placeholders in . Only the first text child used to survive, -// so the copy stopped at the first placeholder and every variable disappeared. -describe("androidResourceExtractor inline markup", () => { - const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" }); - - it("keeps the text and the specifiers around an ", async () => { - const hits = await extract( - ` - We sent it to %1$s. It expires in %2$s minutes. - ` - ); - expect(hits.map((h) => h.value)).toEqual(["We sent it to %1$s. It expires in %2$s minutes."]); - }); - - it("keeps each plural item distinct", async () => { - const hits = await extract( - ` - - Expires in %1$s minute. - Expires in %1$s minutes. - - ` - ); - expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([ - { v: "Expires in %1$s minute.", ids: ["hint", "one"] }, - { v: "Expires in %1$s minutes.", ids: ["hint", "other"] }, - ]); - }); - - it("keeps the words inside an inline style tag", async () => { - const hits = await extract(`Please do not share it`); - expect(hits.map((h) => h.value)).toEqual(["Please do not share it"]); - }); - - it("decodes XML entities", async () => { - const hits = await extract(`<- Back & forth`); - expect(hits.map((h) => h.value)).toEqual(["<- Back & forth"]); - }); -}); diff --git a/lib/src/scan/lang/extractors/android-resources.ts b/lib/src/scan/lang/extractors/android-resources.ts deleted file mode 100644 index 0284d94..0000000 --- a/lib/src/scan/lang/extractors/android-resources.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { Lang, parse, type SgNode } from "@ast-grep/napi"; - -import type { ExtractedHit, LanguageExtractor } from "../types"; -import { decodeEscapes, offsetToLineCol } from "./util"; -import { elementAttribute, emitTextHit, findCdataElements, tagName } from "./xml"; - -/** - * Android `res/values/*.xml` resource files. Three top-level shapes: - * - * Hello, world! - * - * %d item - * %d items - * - * - * Mercury - * Venus - * - * - * Every value is intentional product copy, so all hits flow out as - * `resource_value`. The resource key (and the variant for plurals or the - * index for arrays) is surfaced through `identifiers` so downstream sees - * what's going on without re-parsing. - */ -export const androidResourceExtractor: LanguageExtractor = { - async extract({ source }) { - const root = parse(Lang.Html, source).root(); - const out: ExtractedHit[] = []; - - for (const el of root.findAll({ rule: { kind: "element" } })) { - const tag = tagName(el); - if (tag === "string") { - emitStringElement(el, out, source); - } else if (tag === "plurals" || tag === "string-array") { - const parentName = elementAttribute(el, "name") ?? ""; - let index = 0; - for (const child of el.children()) { - if (child.kind() !== "element" || tagName(child) !== "item") continue; - const variant = tag === "plurals" ? elementAttribute(child, "quantity") ?? "" : String(index); - // The resource name is the lookup key; quantity/index are selectors. - emitTextHit(child, [parentName, variant], out, source, parentName || undefined, decodeEscapes); - index++; - } - } - } - - emitCdataValues(source, out); - - return out; - }, -}; - -const NAME_ATTR_RE = /\bname\s*=\s*"([^"]*)"/; -const QUANTITY_ATTR_RE = /\bquantity\s*=\s*"([^"]*)"/; - -function emitCdataValues(source: string, out: ExtractedHit[]): void { - for (const m of findCdataElements(source, ["string", "item"])) { - const name = NAME_ATTR_RE.exec(m.attrsRaw)?.[1]; - const quantity = QUANTITY_ATTR_RE.exec(m.attrsRaw)?.[1]; - const parent = m.tag === "item" ? findEnclosingResourceParent(source, m.offset) : null; - if (m.tag === "item" && !parent) continue; - const { line, column } = offsetToLineCol(source, m.valueOffset); - out.push({ - value: decodeEscapes(m.value), - location: { line, column }, - context: { - parentRole: "resource_value", - identifiers: buildItemIdentifiers({ name, quantity, parent }), - }, - i18nKey: name || parent || undefined, - }); - } -} - -function buildItemIdentifiers({ - name, - quantity, - parent, -}: { - name: string | undefined; - quantity: string | undefined; - parent: string | null; -}): string[] { - // `` CDATA: name carries the identity directly. - if (name) return quantity ? [name, quantity] : [name]; - // `` CDATA inside /: use the enclosing parent's - // name so the identifier matches the non-CDATA path's [parentName, variant]. - // The array-item index isn't reconstructable from a regex sweep, so plain - // items collapse to just [parentName]. - if (parent) return quantity ? [parent, quantity] : [parent]; - return quantity ? [quantity] : []; -} - -// Walks open/close tags up to `before` and returns the `name` of the -// innermost active ``/`` ancestor, or null if there -// is no enclosing one. -function findEnclosingResourceParent(source: string, before: number): string | null { - const re = /<(plurals|string-array)\b([^>]*)>|<\/(plurals|string-array)>/g; - const stack: string[] = []; - let m: RegExpExecArray | null; - while ((m = re.exec(source)) !== null) { - if (m.index >= before) break; - if (m[1]) stack.push(NAME_ATTR_RE.exec(m[2])?.[1] ?? ""); - else stack.pop(); - } - return stack[stack.length - 1] ?? null; -} - -function emitStringElement(el: SgNode, out: ExtractedHit[], source: string): void { - const name = elementAttribute(el, "name"); - emitTextHit(el, name ? [name] : [], out, source, name ?? undefined, decodeEscapes); -} diff --git a/lib/src/scan/lang/extractors/arb.ts b/lib/src/scan/lang/extractors/arb.ts deleted file mode 100644 index a43dd7e..0000000 --- a/lib/src/scan/lang/extractors/arb.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { LanguageExtractor } from "../types"; -import { jsonI18nExtractor } from "./json-i18n"; - -/** - * Flutter Application Resource Bundles (`.arb`). ARB is plain JSON with one - * convention: keys prefixed with `@` are metadata, not product copy. - * - * { - * "@@locale": "en", - * "hello": "Hello", - * "@hello": { "description": "greeting", "placeholders": {...} } - * } - * - * `@@locale` / `@@last_modified` are file-level metadata; `@` holds the - * description and placeholder spec for ``. Reuse the JSON walker and - * drop any hit whose identifier chain crosses an `@`-prefixed key. - */ -export const arbExtractor: LanguageExtractor = { - async extract(opts) { - const hits = await jsonI18nExtractor.extract(opts); - return hits.filter((h) => !h.context.identifiers.some((id) => id.startsWith("@"))); - }, -}; diff --git a/lib/src/scan/lang/extractors/fallback.test.ts b/lib/src/scan/lang/extractors/fallback.test.ts deleted file mode 100644 index b4982be..0000000 --- a/lib/src/scan/lang/extractors/fallback.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { fallbackExtractor } from "./fallback"; - -const extract = (source: string) => fallbackExtractor.extract({ source, kind: "regex_fallback" }); - -describe("fallbackExtractor", () => { - test("emits one hit per quoted literal with 1-based line/column", async () => { - const hits = await extract(`const greeting = "Hello, world";\n`); - expect(hits).toEqual([ - { - value: "Hello, world", - location: { line: 1, column: 18 }, - context: { parentRole: "other", identifiers: [] }, - }, - ]); - }); - - test("handles single, double, and backtick quotes", async () => { - const hits = await extract(`x = "a"\ny = 'b'\nz = \`c\`\n`); - expect(hits.map((h) => h.value)).toEqual(["a", "b", "c"]); - expect(hits.map((h) => h.location.line)).toEqual([1, 2, 3]); - }); - - test("emits multiple literals on the same line", async () => { - const hits = await extract(`pair("a", "b")\n`); - expect(hits).toHaveLength(2); - expect(hits[0].value).toBe("a"); - expect(hits[1].value).toBe("b"); - expect(hits[0].location.column).toBeLessThan(hits[1].location.column); - }); - - test("respects escaped quotes inside a literal", async () => { - const hits = await extract(`msg = "He said \\"hi\\""\n`); - expect(hits).toHaveLength(1); - expect(hits[0].value).toBe('He said \\"hi\\"'); - }); - - test("returns an empty array when no literals are present", async () => { - const hits = await extract(`const x = 42;\n// no strings here\n`); - expect(hits).toEqual([]); - }); -}); diff --git a/lib/src/scan/lang/extractors/fallback.ts b/lib/src/scan/lang/extractors/fallback.ts deleted file mode 100644 index e1973e3..0000000 --- a/lib/src/scan/lang/extractors/fallback.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { ExtractedHit, LanguageExtractor } from "../types"; - -/** - * Regex-based string finder used for any source file that isn't supported by - * one of our extractors. - * - * Scan each line for `"..."`, `'...'`, or `` `...` `` literals - * and emit them with `parentRole: "other"`. The LLM - * phase reads `source_context` to decide whether each hit is user-facing. - * - * Known limitations - anything caught here is acceptable noise that - * downstream filters or the LLM will handle: - * - Multi-line strings (Python triple-quoted, Go backticks, etc.) get - * truncated at the first newline. - * - Block comments are not stripped; string-like content inside `\/* *\/` - * or `"""..."""` leaks through. - * - Line comments are not stripped either — strings inside `# foo`, - * `// foo`, `-- foo` leak through. (Stripping these would also drop - * real strings like `"#abc"` or `"https://..."`, so we don't bother.) - * - Escape sequences inside strings are matched but not interpreted. - */ -const STRING_RE = /(["'`])((?:(?!\1)[^\\\n]|\\.)*)\1/g; - -export const fallbackExtractor: LanguageExtractor = { - async extract({ source }) { - const out: ExtractedHit[] = []; - const lines = source.split(/\r?\n/); - - for (let i = 0; i < lines.length; i++) { - STRING_RE.lastIndex = 0; - let m: RegExpExecArray | null; - while ((m = STRING_RE.exec(lines[i])) !== null) { - out.push({ - value: m[2], - location: { line: i + 1, column: m.index + 1 }, - context: { parentRole: "other", identifiers: [] }, - }); - } - } - - return out; - }, -}; diff --git a/lib/src/scan/lang/extractors/html-markup.test.ts b/lib/src/scan/lang/extractors/html-markup.test.ts deleted file mode 100644 index 58de204..0000000 --- a/lib/src/scan/lang/extractors/html-markup.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { htmlMarkupExtractor } from "./html-markup"; - -const extract = (source: string) => htmlMarkupExtractor.extract({ source, kind: "html" }); - -describe("htmlMarkupExtractor", () => { - test("emits element text as markup_text tagged with its parent tag", async () => { - const hits = await extract(`

Welcome home

\n`); - const text = hits.find((h) => h.value.trim() === "Welcome home"); - expect(text?.context.parentRole).toBe("markup_text"); - expect(text?.context.parentTag).toBe("p"); - }); - - test("emits plain attribute values as markup_attr with attribute name", async () => { - const hits = await extract(`\n`); - const email = hits.find((h) => h.value === "Email"); - expect(email?.context.parentRole).toBe("markup_attr"); - expect(email?.context.identifiers).toEqual(["placeholder"]); - }); - - test("skips contents of \n

Real copy

\n` - ); - const values = hits.map((h) => h.value.trim()); - expect(values).toContain("Real copy"); - expect(values).not.toContain("Hi"); - }); - - test("skips framework directive attributes (`@click`, `:foo`, `v-on`)", async () => { - const hits = await extract(`\n`); - const attrNames = hits.filter((h) => h.context.parentRole === "markup_attr").flatMap((h) => h.context.identifiers); - expect(attrNames).not.toContain("@click"); - expect(attrNames).not.toContain(":class"); - expect(attrNames).not.toContain("v-bind:id"); - expect(hits.some((h) => h.value.trim() === "Go")).toBe(true); - }); - - test("suppresses parentTag for descendants of /
", async () => {
-    const hits = await extract(`
foo bar
\n`);
-    const inner = hits.find((h) => h.value.trim() === "foo bar");
-    expect(inner?.context.parentRole).toBe("markup_text");
-    expect(inner?.context.parentTag).toBeUndefined();
-  });
-
-  test("extracts inner literals from template interpolation expressions", async () => {
-    const hits = await extract(`
{{ isFollowing ? 'Unfollow' : 'Follow' }}
\n`);
-    const values = hits.filter((h) => h.context.parentRole === "markup_text").map((h) => h.value);
-    expect(values).toEqual(expect.arrayContaining(["Unfollow", "Follow"]));
-  });
-});
diff --git a/lib/src/scan/lang/extractors/html-markup.ts b/lib/src/scan/lang/extractors/html-markup.ts
deleted file mode 100644
index d0239b8..0000000
--- a/lib/src/scan/lang/extractors/html-markup.ts
+++ /dev/null
@@ -1,181 +0,0 @@
-import { Lang, parse, type SgNode } from "@ast-grep/napi";
-
-import type { ExtractedHit, LanguageExtractor } from "../types";
-
-// Tags whose contents are code-shaped, not human copy. If the candidate
-// sits inside any of these up the ancestor chain we suppress parentTag so
-// the markup_text accept rule falls through to the LLM.
-const CODE_LIKE_HTML_ANCESTORS: ReadonlySet = new Set(["pre", "code", "kbd", "samp", "var", "script", "style"]);
-const INTERPOLATION_LITERAL_RE = /(["'`])((?:\\.|(?!\1).)+?)\1/g;
-
-export const htmlMarkupExtractor: LanguageExtractor = {
-  async extract({ source }) {
-    return extractHtmlMarkup(source);
-  },
-};
-
-export function extractHtmlMarkup(source: string): ExtractedHit[] {
-  const root = parse(Lang.Html, source).root();
-  const out: ExtractedHit[] = [];
-
-  for (const el of root.findAll({ rule: { kind: "element" } })) {
-    if (isInsideScriptOrStyle(el)) continue;
-    emitElementMarkup(el, out);
-  }
-
-  return out;
-}
-
-/**
- * ``,
-      ``,
-    ].join("\n");
-    const hits = await extract(source);
-
-    const hello = hits.find((h) => h.value.trim() === "Hello world");
-    expect(hello?.context.parentRole).toBe("markup_text");
-    expect(hello?.context.parentTag).toBe("p");
-
-    const fromScript = hits.find((h) => h.value === '"Hi from script"');
-    expect(fromScript?.context.parentRole).toBe("other");
-  });
-
-  test("script positions are reported against the original .vue file", async () => {
-    const source = [``, ``, ``].join("\n");
-    const hits = await extract(source);
-    const fromScript = hits.find((h) => h.value === '"row2"');
-    expect(fromScript?.location.line).toBe(3);
-  });
-
-  test("ignores `, ``].join(
-      "\n"
-    );
-    const hits = await extract(source);
-    const values = hits.map((h) => h.value.trim());
-    expect(values).toContain("Visible");
-    expect(values).not.toContain("hidden");
-  });
-
-  test("returns empty array for an empty SFC", async () => {
-    const hits = await extract(`\n\n`);
-    expect(hits).toEqual([]);
-  });
-});
diff --git a/lib/src/scan/lang/extractors/vue.ts b/lib/src/scan/lang/extractors/vue.ts
deleted file mode 100644
index 470690c..0000000
--- a/lib/src/scan/lang/extractors/vue.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-import { Lang, parse, type SgNode } from "@ast-grep/napi";
-
-import type { ExtractedHit, LanguageExtractor } from "../types";
-import { extractHtmlMarkup } from "./html-markup";
-import { javascriptExtractor } from "./javascript";
-
-/**
- * Vue Single-File Component (SFC) extractor. A `.vue` file has three
- * top-level blocks:
- *
- *