diff --git a/README.md b/README.md index bc6dcac..0f3f431 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ query surfaces answer with empty results and a note until an index exists. Document extraction and previews use whatever tools are present: `brew install --cask libreoffice` provides `soffice` for DOCX/PPTX previews, and -`pip install python-docx python-pptx openpyxl pymupdf` enables text -extraction for indexing. The Apptainer container build below is +Word, PowerPoint and Excel files extract to Markdown through +`@giraffesyo/downmark`, bundled with the server, so they need nothing extra. The Apptainer container build below is Linux-only. ## Container build diff --git a/deploy/app.def b/deploy/app.def index e396288..b440a81 100644 --- a/deploy/app.def +++ b/deploy/app.def @@ -115,24 +115,25 @@ Stage: final %post # Fail the build on a failed step. An image that quietly lacks an - # extractor indexes Word files as their filename and says nothing, - # which is how the first attempt at this shipped: the apt line named a - # package bookworm does not have, so it installed none of them and the - # build still reported success. + # extractor indexes documents as their filename and says nothing, which + # is how the first attempt at this shipped: the apt line named a package + # bookworm does not have, so it installed none of them and the build + # still reported success. set -e export DEBIAN_FRONTEND=noninteractive # bookworm's python3 is 3.11, which satisfies the indexer's 3.10+ syntax. - # Debian carries docx and openpyxl; python-pptx is not packaged, so it - # comes from pip at build time, never at run time. + # Office documents convert through downmark inside the server bundle, so + # no Python document libraries are needed; poppler and tesseract cover + # PDFs and OCR, LibreOffice renders office pages for previews. apt-get update && apt-get install -y --no-install-recommends \ - python3 python3-pip python3-docx python3-openpyxl \ + python3 \ attr poppler-utils tesseract-ocr curl ca-certificates \ libreoffice-writer libreoffice-impress libreoffice-calc \ fonts-liberation fonts-dejavu-core - pip3 install --no-cache-dir --break-system-packages python-pptx rm -rf /var/lib/apt/lists/* - # Prove it before the image is sealed. - python3 -c "import docx, pptx, openpyxl; print('extractors ok: docx, pptx, openpyxl')" + # Prove the extractors exist before the image is sealed. + command -v pdftotext && command -v tesseract && command -v soffice \ + && ls /app/server/node_modules/@giraffesyo/downmark/dist/downmark.wasm cat > /app/start.sh << 'START' #!/bin/sh # The index lives on the bind-mounted data directory, never in the image; diff --git a/deploy/workflow.yaml b/deploy/workflow.yaml index 2c64060..cb812fe 100644 --- a/deploy/workflow.yaml +++ b/deploy/workflow.yaml @@ -717,47 +717,6 @@ jobs: ssh: remoteHost: ${{ inputs.resource_and_execution.resource.ip }} steps: - - name: Python environment for document extraction - run: | - if [ "${{ inputs.session_settings.session_method }}" = "cleanup" ]; then exit 0; fi - if [ "${{ inputs.app_settings.source }}" = "container" ]; then - echo "Container mode; the image carries the extractors." - exit 0 - fi - cd ${{ inputs.app_settings.workdir }} - # Word, PowerPoint and Excel files index as their filename alone - # without these. The indexer has a standard-library fallback, but - # the libraries read the documents properly, so install them once - # into a venv beside the app and point the indexer at it. - PY="" - for cand in python3.13 python3.12 python3.11 python3.10 python3; do - if command -v "$cand" >/dev/null 2>&1 && "$cand" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then - PY="$cand"; break - fi - done - if [ -z "$PY" ]; then - echo "WARNING: no python 3.10+ found; extraction falls back to the standard library." - exit 0 - fi - if [ ! -x pyenv/bin/python ]; then - "$PY" -m venv pyenv || echo "WARNING: could not create the venv" - fi - if [ -x pyenv/bin/python ]; then - ./pyenv/bin/python -m pip install --quiet --upgrade pip 2>/dev/null || true - ./pyenv/bin/python -m pip install --quiet python-docx python-pptx openpyxl \ - || echo "WARNING: could not install the extractors (no network?); the standard-library fallback applies." - ./pyenv/bin/python - <<'CHECK' - mods = [] - for name, label in (('docx', 'python-docx'), ('pptx', 'python-pptx'), ('openpyxl', 'openpyxl')): - try: - __import__(name) - mods.append(f'{label} ok') - except ImportError: - mods.append(f'{label} MISSING') - print('document extraction:', ', '.join(mods)) - CHECK - fi - - name: Build GUFI from bundled source run: | if [ "${{ inputs.session_settings.session_method }}" = "cleanup" ]; then exit 0; fi @@ -829,8 +788,6 @@ jobs: export INDEX_BASE="${{ inputs.kb_settings.index_base }}" [ -n "$INDEX_BASE" ] || INDEX_BASE="$PWD/app/index" export INDEX_BASE - # Document extraction runs under the venv when one was built. - if [ -x pyenv/bin/python ]; then export PYTHON_BIN="$PWD/pyenv/bin/python"; fi # The embedding model ships inside the app tree; a relocated index # needs it at $INDEX_BASE/models. if [ "$INDEX_BASE" != "$PWD/app/index" ] && [ ! -f "$INDEX_BASE/models/minilm384.gguf" ] && [ -f app/index/models/minilm384.gguf ]; then @@ -910,9 +867,6 @@ jobs: if [ -x "${WORKDIR}/soffice-wrapper.sh" ]; then echo "export SOFFICE_BIN=${WORKDIR}/soffice-wrapper.sh" >> ${SCRIPT} fi - if [ -x "${WORKDIR}/pyenv/bin/python" ]; then - echo "export PYTHON_BIN=${WORKDIR}/pyenv/bin/python" >> ${SCRIPT} - fi if [ -n "${{ inputs.kb_settings.index_base }}" ]; then echo "export INDEX_BASE='${{ inputs.kb_settings.index_base }}'" >> ${SCRIPT} else diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f2abf50..3affc1d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -23,7 +23,7 @@ CREATE VIRTUAL TABLE words USING fts5(tinode UNINDEXED, fname UNINDEXED, wordf) One row per file: `tinode` is the source file's inode as text, `fname` its name, `wordf` the extracted text (capped at 500 KB). The inode is text rather than an integer because parallel filesystems issue inode numbers above SQLite's signed 64-bit limit, which raised OverflowError and failed the whole pass; every consumer joins on `CAST(tinode AS TEXT)` anyway, since GUFI stores `entries.inode` as TEXT and an uncast comparison matches nothing. -Extraction is by suffix. Markdown, text, code, and config files are read directly. DOCX, PPTX and XLSX use python-docx, python-pptx and openpyxl, falling back to a standard-library reader when those are missing, since the three formats are zipped XML and a host that cannot install packages can still read them. PDFs use `pdftotext -layout`, which keeps columns and table cells apart, with pypdf as a fallback; a PDF whose text layer is thin for its page count is a scan, so its pages are rendered with `pdftoppm` and read with tesseract. Extracted text is written to `$INDEX_BASE/extract/.txt` and reused for instant previews. +Extraction is by suffix. Markdown, text, code, and config files are read directly. DOCX, PPTX, XLSX and legacy DOC convert to Markdown through [downmark](https://github.com/giraffesyo/downmark), a pure-Go document converter shipped as WebAssembly in the server's npm dependencies (`@giraffesyo/downmark`), so Office extraction needs nothing from the host: `server/src/preextract.ts` runs as a child process before each `enrich.py` pass (and from `reindex.sh`) and writes the cache entries `enrich.py` then reuses. Headings, tables, slide boundaries, speaker notes, chart data and equations survive, and the viewer renders the result. Should downmark fail on a document, `enrich.py` falls back to a standard-library reader, since the three formats are zipped XML; no Python document libraries are involved. Cache entries are refreshed once per downmark version (`extract/.downmark-version`). PDFs use `pdftotext -layout`, which keeps columns and table cells apart, with pypdf as a fallback; a PDF whose text layer is thin for its page count is a scan, so its pages are rendered with `pdftoppm` and read with tesseract. Extracted text is written to `$INDEX_BASE/extract/.txt` and reused for instant previews. Expensive extractions are cached by source mtime. Only images cache an empty result, where a picture with no text is a real answer; a document that extracted to nothing is retried on the next pass, so it indexes once the reader for its format is installed. `GET /api/index/extractors` reports which readers this host has, and the Stats page names any format that can only be indexed by filename. diff --git a/indexer/enrich.py b/indexer/enrich.py index 16bf240..41e1574 100644 --- a/indexer/enrich.py +++ b/indexer/enrich.py @@ -7,6 +7,12 @@ text is also written to an on-disk extract cache the server uses for previews. Every directory db gets the `words` table, empty or not, so MATCH queries never hit a missing table. + +Office documents normally arrive already converted: the server (and +reindex.sh) run `server/dist/preextract.js`, which turns .docx/.pptx/.xlsx/ +.doc into Markdown through downmark and writes the cache entries this script +reuses. The standard-library OOXML reader below is the last resort for a +document that pass could not read; PDFs, OCR and image captions live here. """ import argparse import os @@ -23,7 +29,8 @@ '.md', '.txt', '.py', '.sh', '.yaml', '.yml', '.json', '.csv', '.tsv', '.xml', '.html', '.css', '.js', '.ts', '.tsx', '.svg', '.toml', '.cfg', '.ini', '.def', '.mjs', '.sql', } -EXTRACT_SUFFIXES = {'.pdf', '.docx', '.pptx', '.xlsx'} +# .doc has no reader here; it indexes only through the pre-extracted cache. +EXTRACT_SUFFIXES = {'.pdf', '.docx', '.pptx', '.xlsx', '.doc'} IMAGE_SUFFIXES = {'.png', '.jpg', '.jpeg', '.gif', '.webp'} MAX_TEXT = 500_000 MAX_CAPTION_BYTES = 8 * 1024 * 1024 @@ -33,10 +40,10 @@ def _ooxml_text(path: Path, members: str, break_tags: tuple) -> str: """Text out of an OOXML file with the standard library alone. - docx, pptx and xlsx are zipped XML, so the text is reachable without - python-docx, python-pptx or openpyxl. Clusters routinely have none of - them and no way to install them, and a Word document that indexes as - its filename is worse than a slightly rougher extraction. + docx, pptx and xlsx are zipped XML, so their text is reachable without + any library. This only runs for a document downmark could not convert + (see the module docstring): a rough extraction beats indexing a Word + document as its filename. """ import re import zipfile @@ -58,16 +65,7 @@ def _ooxml_text(path: Path, members: str, break_tags: tuple) -> str: def extract_docx(path: Path) -> str: - try: - from docx import Document # type: ignore - except ImportError: - return _ooxml_text(path, r'word/document\.xml', ('', '', '')) - doc = Document(str(path)) - parts = [p.text for p in doc.paragraphs] - for table in doc.tables: - for row in table.rows: - parts.append('\t'.join(c.text for c in row.cells)) - return '\n'.join(x for x in parts if x and x.strip()) + return _ooxml_text(path, r'word/document\.xml', ('', '', '')) # A PDF with almost no extractable text is a scan; below this many @@ -147,34 +145,12 @@ def extract_pdf(path: Path) -> str: def extract_pptx(path: Path) -> str: - try: - from pptx import Presentation # type: ignore - except ImportError: - return _ooxml_text(path, r'ppt/slides/slide\d+\.xml', ('', '')) - prs = Presentation(str(path)) - parts = [] - for slide in prs.slides: - for shape in slide.shapes: - if shape.has_text_frame: - parts.append(shape.text_frame.text) - return '\n'.join(x for x in parts if x and x.strip()) + return _ooxml_text(path, r'ppt/slides/slide\d+\.xml', ('', '')) def extract_xlsx(path: Path) -> str: - try: - from openpyxl import load_workbook # type: ignore - except ImportError: - # Shared strings hold most cell text; sheet XML holds the rest. - return _ooxml_text(path, r'xl/(sharedStrings|worksheets/sheet\d+)\.xml', ('', '', '')) - wb = load_workbook(str(path), read_only=True, data_only=True) - parts = [] - for ws in wb.worksheets: - parts.append(f'# sheet: {ws.title}') - for row in ws.iter_rows(values_only=True): - cells = [str(c) for c in row if c is not None] - if cells: - parts.append('\t'.join(cells)) - return '\n'.join(parts) + # Shared strings hold most cell text; sheet XML holds the rest. + return _ooxml_text(path, r'xl/(sharedStrings|worksheets/sheet\d+)\.xml', ('', '', '')) def gateway_key() -> str: diff --git a/indexer/reindex.sh b/indexer/reindex.sh index 572755b..2aed83d 100755 --- a/indexer/reindex.sh +++ b/indexer/reindex.sh @@ -34,6 +34,18 @@ fi PYTHON_BIN="${PYTHON_BIN:-python3}" echo "python: $PYTHON_BIN ($($PYTHON_BIN --version 2>&1))" +# Office documents convert to Markdown through downmark, bundled with the +# server, before enrichment; enrich.py reuses those cache entries and keeps +# its own readers for anything that pass skipped. Needs the built server. +PREEXTRACT="$PROJECT_ROOT/server/dist/preextract.js" +NODE_BIN="${NODE_BIN:-node}" +if [ -f "$PREEXTRACT" ] && command -v "$NODE_BIN" >/dev/null 2>&1; then + "$NODE_BIN" "$PREEXTRACT" --kb-root "$KB_ROOT" --extract-cache "$INDEX_BASE/extract" \ + || echo "WARNING: office pre-extraction failed; enrich.py uses its own readers" +else + echo "server/dist/preextract.js or node not found; enrich.py uses its own office readers" +fi + "$PYTHON_BIN" "$PROJECT_ROOT/indexer/enrich.py" --kb-root "$KB_ROOT" \ --index "$STAGING/$(basename "$KB_ROOT")" --extract-cache "$INDEX_BASE/extract" diff --git a/indexer/requirements.txt b/indexer/requirements.txt index 62270d0..3e1214c 100644 --- a/indexer/requirements.txt +++ b/indexer/requirements.txt @@ -1,4 +1,2 @@ -python-docx +# Optional: PDF text fallback for hosts without poppler (pdftotext). pypdf -python-pptx -openpyxl diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c74c736..b9f2dfe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,6 +16,9 @@ importers: '@fastify/static': specifier: ^8.0.0 version: 8.3.0 + '@giraffesyo/downmark': + specifier: 0.6.0 + version: 0.6.0 '@parallelworks/workflow-parser': specifier: ^0.1.0 version: 0.1.0 @@ -548,6 +551,10 @@ packages: '@fontsource-variable/geist@5.3.0': resolution: {integrity: sha512-j0m+vLQuG5XAYoHtGCVu0spvlGreR3EzpECUVzkFmI1mTVnAO38l/NEPDCFgZ177JxzYJCLSmTQibIiYPilGrA==} + '@giraffesyo/downmark@0.6.0': + resolution: {integrity: sha512-dXYZWnOZY1Q0+9am/qubRiKWlBeblvNRaDI4ZJr6fb7IMrMIUXY3WgP+C/1wBzZL8FPg6EEFEY6aBmKpXcvXYw==} + engines: {node: '>=18'} + '@headlessui/react@2.2.10': resolution: {integrity: sha512-5pVLNK9wlpxTUTy9GpgbX/SdcRh+HBnPktjM2wbiLTH4p+2EPHBO1aoSryUCuKUIItdDWO9ITlhUL8UnUN/oIA==} engines: {node: '>=10'} @@ -2699,6 +2706,8 @@ snapshots: '@fontsource-variable/geist@5.3.0': {} + '@giraffesyo/downmark@0.6.0': {} + '@headlessui/react@2.2.10(react-dom@19.2.8(react@19.2.8))(react@19.2.8)': dependencies: '@floating-ui/react': 0.26.28(react-dom@19.2.8(react@19.2.8))(react@19.2.8) diff --git a/server/package.json b/server/package.json index 9b98743..120f0ed 100644 --- a/server/package.json +++ b/server/package.json @@ -11,6 +11,7 @@ "dependencies": { "@fastify/multipart": "^10.1.0", "@fastify/static": "^8.0.0", + "@giraffesyo/downmark": "0.6.0", "@parallelworks/workflow-parser": "^0.1.0", "fastify": "^5.0.0", "jose": "^6.2.8", diff --git a/server/src/config.ts b/server/src/config.ts index 95278e8..dce438a 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -81,7 +81,7 @@ export const TEXT_SUFFIXES = new Set([ '.html', '.css', '.js', '.ts', '.tsx', '.svg', '.toml', '.cfg', '.ini', '.def', '.mjs', '.sql', '.log', '.conf', '.env', '.service', ]) -export const EXTRACTED_SUFFIXES = new Set(['.pdf', '.docx', '.pptx', '.xlsx']) +export const EXTRACTED_SUFFIXES = new Set(['.pdf', '.docx', '.pptx', '.xlsx', '.doc']) export const IMAGE_SUFFIXES = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp']) export const MODEL_SUFFIXES = new Set(['.stl', '.step', '.stp']) diff --git a/server/src/indexing.ts b/server/src/indexing.ts index e2058db..c1906e6 100644 --- a/server/src/indexing.ts +++ b/server/src/indexing.ts @@ -4,6 +4,7 @@ import fsp from 'node:fs/promises' import path from 'node:path' import { EXCLUDE_DIRS, GUFI_BIN, GUFI_INDEX, INDEX_BASE, KB_ROOT, PROJECT_ROOT, PYTHON_BIN } from './config.js' import { invalidateDbList } from './gufi.js' +import { runPreExtract } from './preextract.js' import { invalidateContext } from './chat/context.js' import { effectiveSettings } from './settings.js' @@ -44,18 +45,14 @@ let extractorCache: ExtractorReport[] | null = null export function extractorReport(): ExtractorReport[] { if (extractorCache) return extractorCache - const probe = (code: string): boolean => { - try { execFileSync(PYTHON_BIN, ['-c', code], { stdio: 'ignore', timeout: 15_000 }); return true } - catch { return false } - } const which = (bin: string): boolean => { try { execFileSync('sh', ['-c', `command -v ${bin}`], { stdio: 'ignore', timeout: 10_000 }); return true } catch { return false } } + // Office formats convert through downmark, which ships inside the server + // bundle, so they never depend on the host. extractorCache = [ - { name: 'python-docx', available: probe('import docx'), covers: '.docx' }, - { name: 'python-pptx', available: probe('import pptx'), covers: '.pptx' }, - { name: 'openpyxl', available: probe('import openpyxl'), covers: '.xlsx' }, + { name: 'downmark', available: true, covers: '.docx, .pptx, .xlsx, .doc' }, { name: 'pdftotext', available: which('pdftotext'), covers: '.pdf' }, { name: 'tesseract', available: which('tesseract'), covers: 'text inside images (OCR)' }, ] @@ -165,6 +162,8 @@ export function incrementalIndexDir(rel: string): Promise<{ ms: number }> { await fsp.rm(target, { recursive: true, force: true }) await fsp.rename(built, target) + // Office documents go to Markdown first; enrich.py reuses the entries. + await runPreExtract({ kbRoot: KB_ROOT, cacheRoot: path.join(INDEX_BASE, 'extract'), subdir: cleaned }, m => console.warn(m)) await run(PYTHON_BIN, [path.join(PROJECT_ROOT, 'indexer', 'enrich.py'), '--kb-root', KB_ROOT, '--index', GUFI_INDEX, '--extract-cache', path.join(INDEX_BASE, 'extract'), '--subdir', cleaned]) @@ -202,6 +201,7 @@ export function indexRootDb(): Promise<{ ms: number }> { const built = path.join(staging, path.basename(KB_ROOT), 'db.db') await fsp.mkdir(GUFI_INDEX, { recursive: true }) await fsp.rename(built, path.join(GUFI_INDEX, 'db.db')) + await runPreExtract({ kbRoot: KB_ROOT, cacheRoot: path.join(INDEX_BASE, 'extract'), noRecurse: true }, m => console.warn(m)) await run(PYTHON_BIN, [path.join(PROJECT_ROOT, 'indexer', 'enrich.py'), '--kb-root', KB_ROOT, '--index', GUFI_INDEX, '--extract-cache', path.join(INDEX_BASE, 'extract'), '--no-recurse']) diff --git a/server/src/preextract.ts b/server/src/preextract.ts new file mode 100644 index 0000000..2120635 --- /dev/null +++ b/server/src/preextract.ts @@ -0,0 +1,180 @@ +/** + * Office documents to Markdown for the extract cache, ahead of enrich.py. + * + * Word, PowerPoint and Excel files (plus legacy .doc) convert through + * downmark (the pure-Go document converter compiled to WebAssembly, shipped + * as an npm dependency) into `$INDEX_BASE/extract/.txt`. enrich.py + * already reuses any cache entry at least as new as its source, so it picks + * these up as-is and indexes them; a file downmark cannot read gets no entry + * and falls through to enrich.py's standard-library OOXML pass, the only + * reader left there for these formats. PDFs stay with pdftotext and OCR in + * enrich.py; poppler is present for page previews regardless and downmark + * has no OCR. + * + * Markdown rather than flat text because the downstream consumers are the + * chat model, full-text snippets, and the viewer: headings, tables and slide + * boundaries survive, and the viewer renders them. + * + * Runs as a child process (see runPreExtract) or as a CLI from reindex.sh: + * node server/dist/preextract.js --kb-root DIR --extract-cache DIR [--subdir REL] [--no-recurse] + * The wasm is single-threaded and blocks its thread per document, so the + * server never converts on its own event loop. + */ +import { fork } from 'node:child_process' +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +export const OFFICE_MARKDOWN_SUFFIXES = new Set(['.docx', '.pptx', '.xlsx', '.doc']) + +// Same cap as enrich.py's MAX_TEXT: the index holds the head of a huge +// workbook, full-text search covers the rest of nothing. +const MAX_TEXT = 500_000 +// downmark refuses OOXML archives above this anyway; skip the read. +const MAX_INPUT_BYTES = 64 * 1024 * 1024 +// Cache entries written by an older converter (or by enrich.py's flat-text +// readers before this existed) are refreshed once per downmark version; the +// marker is written after a full-scope pass completes. +const VERSION_MARKER = '.downmark-version' + +export interface PreExtractOptions { + kbRoot: string + cacheRoot: string + subdir?: string + noRecurse?: boolean + excludeDirs?: Set + log?: (msg: string) => void +} + +export interface PreExtractSummary { + scanned: number + converted: number + reused: number + failed: number + ms: number +} + +const DEFAULT_EXCLUDE = new Set([ + '.git', 'node_modules', '.venv', 'venv', '__pycache__', 'dist', 'build', + '.cache', '.pytest_cache', 'screenshots', '.ipynb_checkpoints', +]) + +async function readMarker(cacheRoot: string): Promise { + try { return (await fsp.readFile(path.join(cacheRoot, VERSION_MARKER), 'utf8')).trim() } catch { return '' } +} + +/** Convert every Office document under the scope whose cache entry is + * missing or stale. Never throws for a single document; failures are + * counted and logged so enrich.py's readers take over for those files. */ +export async function preExtract(opts: PreExtractOptions): Promise { + const t0 = Date.now() + const log = opts.log ?? (() => {}) + const exclude = opts.excludeDirs ?? DEFAULT_EXCLUDE + const { convert, version } = await import('@giraffesyo/downmark') + const ver = await version() + const refreshAll = (await readMarker(opts.cacheRoot)) !== ver + const summary: PreExtractSummary = { scanned: 0, converted: 0, reused: 0, failed: 0, ms: 0 } + + const sub = (opts.subdir ?? '').replace(/^\/+|\/+$/g, '') + const start = sub ? path.join(opts.kbRoot, sub) : opts.kbRoot + if (!path.resolve(start).startsWith(path.resolve(opts.kbRoot))) throw new Error('subdir escapes the knowledge base') + + const walk = async (dir: string, rel: string): Promise => { + let entries + try { entries = await fsp.readdir(dir, { withFileTypes: true }) } catch { return } + entries.sort((a, b) => a.name.localeCompare(b.name)) + for (const e of entries) { + const abs = path.join(dir, e.name) + const childRel = rel ? `${rel}/${e.name}` : e.name + if (e.isDirectory()) { + if (!opts.noRecurse && !exclude.has(e.name) && !e.name.startsWith('.')) await walk(abs, childRel) + continue + } + if (!e.isFile() || !OFFICE_MARKDOWN_SUFFIXES.has(path.extname(e.name).toLowerCase())) continue + summary.scanned++ + let st: fs.Stats + try { st = await fsp.stat(abs) } catch { continue } + const cacheFile = path.join(opts.cacheRoot, childRel + '.txt') + if (!refreshAll) { + try { + const cst = await fsp.stat(cacheFile) + if (cst.mtimeMs >= st.mtimeMs && cst.size > 0) { summary.reused++; continue } + } catch { /* no entry yet */ } + } + if (st.size > MAX_INPUT_BYTES) { summary.failed++; log(`preextract: ${childRel}: ${st.size} bytes exceeds the converter's input limit`); continue } + try { + const data = await fsp.readFile(abs) + const { markdown } = await convert(data, { filename: e.name }) + const text = markdown.slice(0, MAX_TEXT) + if (!text.trim()) { summary.failed++; log(`preextract: ${childRel}: no text`); continue } + await fsp.mkdir(path.dirname(cacheFile), { recursive: true }) + const tmp = `${cacheFile}.${process.pid}.tmp` + await fsp.writeFile(tmp, text) + await fsp.rename(tmp, cacheFile) + summary.converted++ + } catch (err) { + summary.failed++ + log(`preextract: ${childRel}: ${String((err as Error).message ?? err).slice(0, 200)}`) + } + } + } + await walk(start, sub) + + // Only a pass over the whole tree proves every entry is current. + if (refreshAll && !sub && !opts.noRecurse) { + await fsp.mkdir(opts.cacheRoot, { recursive: true }) + await fsp.writeFile(path.join(opts.cacheRoot, VERSION_MARKER), ver + '\n') + } + summary.ms = Date.now() - t0 + return summary +} + +/** Run preExtract in a child process so a long conversion never blocks the + * server. Resolves to the summary, or null when the child failed; callers + * continue either way because enrich.py covers the same files. */ +export function runPreExtract(opts: Omit, log: (msg: string) => void = () => {}): Promise { + const self = fileURLToPath(import.meta.url) + const script = path.join(path.dirname(self), 'preextract' + path.extname(self)) + const args = ['--kb-root', opts.kbRoot, '--extract-cache', opts.cacheRoot] + if (opts.subdir) args.push('--subdir', opts.subdir) + if (opts.noRecurse) args.push('--no-recurse') + return new Promise(resolve => { + // fork inherits execArgv, so the tsx loader used in development carries + // over; in production the script is plain JS under dist/. + const child = fork(script, args, { stdio: ['ignore', 'pipe', 'pipe', 'ipc'], timeout: 30 * 60_000 }) + let out = '' + let err = '' + child.stdout?.on('data', d => { out += d }) + child.stderr?.on('data', d => { err += d }) + child.on('error', e => { log(`preextract: ${e.message}`); resolve(null) }) + child.on('exit', code => { + for (const line of err.split('\n')) if (line.trim()) log(line.trim()) + if (code !== 0) { log(`preextract: exited ${code}`); resolve(null); return } + try { resolve(JSON.parse(out.trim().split('\n').pop() ?? '')) } catch { resolve(null) } + }) + }) +} + +function parseArgs(argv: string[]): PreExtractOptions { + const opts: PreExtractOptions = { kbRoot: '', cacheRoot: '' } + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (a === '--kb-root') opts.kbRoot = argv[++i] ?? '' + else if (a === '--extract-cache') opts.cacheRoot = argv[++i] ?? '' + else if (a === '--subdir') opts.subdir = argv[++i] ?? '' + else if (a === '--no-recurse') opts.noRecurse = true + else throw new Error(`unknown argument: ${a}`) + } + if (!opts.kbRoot || !opts.cacheRoot) throw new Error('usage: preextract --kb-root DIR --extract-cache DIR [--subdir REL] [--no-recurse]') + return opts +} + +const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href +if (invokedDirectly) { + let opts: PreExtractOptions + try { opts = parseArgs(process.argv.slice(2)) } catch (e) { console.error(String((e as Error).message)); process.exit(2) } + preExtract({ ...opts!, log: m => console.error(m) }) + .then(s => { console.log(JSON.stringify(s)); process.exit(0) }) + .catch(e => { console.error(`preextract: ${String((e as Error).message ?? e)}`); process.exit(1) }) +} diff --git a/web/src/components/Viewer.tsx b/web/src/components/Viewer.tsx index aa33d0c..761d69b 100644 --- a/web/src/components/Viewer.tsx +++ b/web/src/components/Viewer.tsx @@ -18,7 +18,7 @@ const PDF_PAGE_CAP = 150 /** PDFs and office docs preview as server-rendered page images; the * browser's own PDF viewer is unavailable in the platform's sandboxed * session iframe, so an