feat(indexer): extract Office documents to Markdown with downmark

README.md

@@ -58,8 +58,8 @@ query surfaces answer with empty results and a note until an index
 exists. Document
 extraction and previews use whatever tools are present: `brew install
 --cask libreoffice` provides `soffice` for DOCX/PPTX previews, and
-`pip install python-docx python-pptx openpyxl pymupdf` enables text
-extraction for indexing. The Apptainer container build below is
+Word, PowerPoint and Excel files extract to Markdown through
+`@giraffesyo/downmark`, bundled with the server, so they need nothing extra. The Apptainer container build below is
 Linux-only.
 
 ## Container build

deploy/app.def

@@ -115,24 +115,25 @@ Stage: final
 
 %post
     # Fail the build on a failed step. An image that quietly lacks an
-    # extractor indexes Word files as their filename and says nothing,
-    # which is how the first attempt at this shipped: the apt line named a
-    # package bookworm does not have, so it installed none of them and the
-    # build still reported success.
+    # extractor indexes documents as their filename and says nothing, which
+    # is how the first attempt at this shipped: the apt line named a package
+    # bookworm does not have, so it installed none of them and the build
+    # still reported success.
     set -e
     export DEBIAN_FRONTEND=noninteractive
     # bookworm's python3 is 3.11, which satisfies the indexer's 3.10+ syntax.
-    # Debian carries docx and openpyxl; python-pptx is not packaged, so it
-    # comes from pip at build time, never at run time.
+    # Office documents convert through downmark inside the server bundle, so
+    # no Python document libraries are needed; poppler and tesseract cover
+    # PDFs and OCR, LibreOffice renders office pages for previews.
     apt-get update && apt-get install -y --no-install-recommends \
-        python3 python3-pip python3-docx python3-openpyxl \
+        python3 \
         attr poppler-utils tesseract-ocr curl ca-certificates \
         libreoffice-writer libreoffice-impress libreoffice-calc \
         fonts-liberation fonts-dejavu-core
-    pip3 install --no-cache-dir --break-system-packages python-pptx
     rm -rf /var/lib/apt/lists/*
-    # Prove it before the image is sealed.
-    python3 -c "import docx, pptx, openpyxl; print('extractors ok: docx, pptx, openpyxl')"
+    # Prove the extractors exist before the image is sealed.
+    command -v pdftotext && command -v tesseract && command -v soffice \
+        && ls /app/server/node_modules/@giraffesyo/downmark/dist/downmark.wasm
     cat > /app/start.sh << 'START'
 #!/bin/sh
 # The index lives on the bind-mounted data directory, never in the image;

deploy/workflow.yaml

@@ -717,47 +717,6 @@ jobs:
     ssh:
       remoteHost: ${{ inputs.resource_and_execution.resource.ip }}
     steps:
-      - name: Python environment for document extraction
-        run: |
-          if [ "${{ inputs.session_settings.session_method }}" = "cleanup" ]; then exit 0; fi
-          if [ "${{ inputs.app_settings.source }}" = "container" ]; then
-            echo "Container mode; the image carries the extractors."
-            exit 0
-          fi
-          cd ${{ inputs.app_settings.workdir }}
-          # Word, PowerPoint and Excel files index as their filename alone
-          # without these. The indexer has a standard-library fallback, but
-          # the libraries read the documents properly, so install them once
-          # into a venv beside the app and point the indexer at it.
-          PY=""
-          for cand in python3.13 python3.12 python3.11 python3.10 python3; do
-            if command -v "$cand" >/dev/null 2>&1 && "$cand" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then
-              PY="$cand"; break
-            fi
-          done
-          if [ -z "$PY" ]; then
-            echo "WARNING: no python 3.10+ found; extraction falls back to the standard library."
-            exit 0
-          fi
-          if [ ! -x pyenv/bin/python ]; then
-            "$PY" -m venv pyenv || echo "WARNING: could not create the venv"
-          fi
-          if [ -x pyenv/bin/python ]; then
-            ./pyenv/bin/python -m pip install --quiet --upgrade pip 2>/dev/null || true
-            ./pyenv/bin/python -m pip install --quiet python-docx python-pptx openpyxl \
-              || echo "WARNING: could not install the extractors (no network?); the standard-library fallback applies."
-            ./pyenv/bin/python - <<'CHECK'
-          mods = []
-          for name, label in (('docx', 'python-docx'), ('pptx', 'python-pptx'), ('openpyxl', 'openpyxl')):
-              try:
-                  __import__(name)
-                  mods.append(f'{label} ok')
-              except ImportError:
-                  mods.append(f'{label} MISSING')
-          print('document extraction:', ', '.join(mods))
-          CHECK
-          fi
-
       - name: Build GUFI from bundled source
         run: |
           if [ "${{ inputs.session_settings.session_method }}" = "cleanup" ]; then exit 0; fi
@@ -829,8 +788,6 @@ jobs:
           export INDEX_BASE="${{ inputs.kb_settings.index_base }}"
           [ -n "$INDEX_BASE" ] || INDEX_BASE="$PWD/app/index"
           export INDEX_BASE
-          # Document extraction runs under the venv when one was built.
-          if [ -x pyenv/bin/python ]; then export PYTHON_BIN="$PWD/pyenv/bin/python"; fi
           # The embedding model ships inside the app tree; a relocated index
           # needs it at $INDEX_BASE/models.
           if [ "$INDEX_BASE" != "$PWD/app/index" ] && [ ! -f "$INDEX_BASE/models/minilm384.gguf" ] && [ -f app/index/models/minilm384.gguf ]; then
@@ -910,9 +867,6 @@ jobs:
           if [ -x "${WORKDIR}/soffice-wrapper.sh" ]; then
             echo "export SOFFICE_BIN=${WORKDIR}/soffice-wrapper.sh" >> ${SCRIPT}
           fi
-          if [ -x "${WORKDIR}/pyenv/bin/python" ]; then
-            echo "export PYTHON_BIN=${WORKDIR}/pyenv/bin/python" >> ${SCRIPT}
-          fi
           if [ -n "${{ inputs.kb_settings.index_base }}" ]; then
             echo "export INDEX_BASE='${{ inputs.kb_settings.index_base }}'" >> ${SCRIPT}
           else

docs/ARCHITECTURE.md

@@ -23,7 +23,7 @@ CREATE VIRTUAL TABLE words USING fts5(tinode UNINDEXED, fname UNINDEXED, wordf)
 
 One row per file: `tinode` is the source file's inode as text, `fname` its name, `wordf` the extracted text (capped at 500 KB). The inode is text rather than an integer because parallel filesystems issue inode numbers above SQLite's signed 64-bit limit, which raised OverflowError and failed the whole pass; every consumer joins on `CAST(tinode AS TEXT)` anyway, since GUFI stores `entries.inode` as TEXT and an uncast comparison matches nothing.
 
-Extraction is by suffix. Markdown, text, code, and config files are read directly. DOCX, PPTX and XLSX use python-docx, python-pptx and openpyxl, falling back to a standard-library reader when those are missing, since the three formats are zipped XML and a host that cannot install packages can still read them. PDFs use `pdftotext -layout`, which keeps columns and table cells apart, with pypdf as a fallback; a PDF whose text layer is thin for its page count is a scan, so its pages are rendered with `pdftoppm` and read with tesseract. Extracted text is written to `$INDEX_BASE/extract/<relpath>.txt` and reused for instant previews.
+Extraction is by suffix. Markdown, text, code, and config files are read directly. DOCX, PPTX, XLSX and legacy DOC convert to Markdown through [downmark](https://github.com/giraffesyo/downmark), a pure-Go document converter shipped as WebAssembly in the server's npm dependencies (`@giraffesyo/downmark`), so Office extraction needs nothing from the host: `server/src/preextract.ts` runs as a child process before each `enrich.py` pass (and from `reindex.sh`) and writes the cache entries `enrich.py` then reuses. Headings, tables, slide boundaries, speaker notes, chart data and equations survive, and the viewer renders the result. Should downmark fail on a document, `enrich.py` falls back to a standard-library reader, since the three formats are zipped XML; no Python document libraries are involved. Cache entries are refreshed once per downmark version (`extract/.downmark-version`). PDFs use `pdftotext -layout`, which keeps columns and table cells apart, with pypdf as a fallback; a PDF whose text layer is thin for its page count is a scan, so its pages are rendered with `pdftoppm` and read with tesseract. Extracted text is written to `$INDEX_BASE/extract/<relpath>.txt` and reused for instant previews.
 
 Expensive extractions are cached by source mtime. Only images cache an empty result, where a picture with no text is a real answer; a document that extracted to nothing is retried on the next pass, so it indexes once the reader for its format is installed. `GET /api/index/extractors` reports which readers this host has, and the Stats page names any format that can only be indexed by filename.
 

indexer/enrich.py

@@ -7,6 +7,12 @@
 text is also written to an on-disk extract cache the server uses for
 previews. Every directory db gets the `words` table, empty or not, so
 MATCH queries never hit a missing table.
+
+Office documents normally arrive already converted: the server (and
+reindex.sh) run `server/dist/preextract.js`, which turns .docx/.pptx/.xlsx/
+.doc into Markdown through downmark and writes the cache entries this script
+reuses. The standard-library OOXML reader below is the last resort for a
+document that pass could not read; PDFs, OCR and image captions live here.
 """
 import argparse
 import os
@@ -23,7 +29,8 @@
     '.md', '.txt', '.py', '.sh', '.yaml', '.yml', '.json', '.csv', '.tsv', '.xml',
     '.html', '.css', '.js', '.ts', '.tsx', '.svg', '.toml', '.cfg', '.ini', '.def', '.mjs', '.sql',
 }
-EXTRACT_SUFFIXES = {'.pdf', '.docx', '.pptx', '.xlsx'}
+# .doc has no reader here; it indexes only through the pre-extracted cache.
+EXTRACT_SUFFIXES = {'.pdf', '.docx', '.pptx', '.xlsx', '.doc'}
 IMAGE_SUFFIXES = {'.png', '.jpg', '.jpeg', '.gif', '.webp'}
 MAX_TEXT = 500_000
 MAX_CAPTION_BYTES = 8 * 1024 * 1024
@@ -33,10 +40,10 @@
 def _ooxml_text(path: Path, members: str, break_tags: tuple) -> str:
     """Text out of an OOXML file with the standard library alone.
 
-    docx, pptx and xlsx are zipped XML, so the text is reachable without
-    python-docx, python-pptx or openpyxl. Clusters routinely have none of
-    them and no way to install them, and a Word document that indexes as
-    its filename is worse than a slightly rougher extraction.
+    docx, pptx and xlsx are zipped XML, so their text is reachable without
+    any library. This only runs for a document downmark could not convert
+    (see the module docstring): a rough extraction beats indexing a Word
+    document as its filename.
     """
     import re
     import zipfile
@@ -58,16 +65,7 @@ def _ooxml_text(path: Path, members: str, break_tags: tuple) -> str:
 
 
 def extract_docx(path: Path) -> str:
-    try:
-        from docx import Document  # type: ignore
-    except ImportError:
-        return _ooxml_text(path, r'word/document\.xml', ('</w:p>', '</w:tr>', '</w:tc>'))
-    doc = Document(str(path))
-    parts = [p.text for p in doc.paragraphs]
-    for table in doc.tables:
-        for row in table.rows:
-            parts.append('\t'.join(c.text for c in row.cells))
-    return '\n'.join(x for x in parts if x and x.strip())
+    return _ooxml_text(path, r'word/document\.xml', ('</w:p>', '</w:tr>', '</w:tc>'))
 
 
 # A PDF with almost no extractable text is a scan; below this many
@@ -147,34 +145,12 @@ def extract_pdf(path: Path) -> str:
 
 
 def extract_pptx(path: Path) -> str:
-    try:
-        from pptx import Presentation  # type: ignore
-    except ImportError:
-        return _ooxml_text(path, r'ppt/slides/slide\d+\.xml', ('</a:p>', '</a:t>'))
-    prs = Presentation(str(path))
-    parts = []
-    for slide in prs.slides:
-        for shape in slide.shapes:
-            if shape.has_text_frame:
-                parts.append(shape.text_frame.text)
-    return '\n'.join(x for x in parts if x and x.strip())
+    return _ooxml_text(path, r'ppt/slides/slide\d+\.xml', ('</a:p>', '</a:t>'))
 
 
 def extract_xlsx(path: Path) -> str:
-    try:
-        from openpyxl import load_workbook  # type: ignore
-    except ImportError:
-        # Shared strings hold most cell text; sheet XML holds the rest.
-        return _ooxml_text(path, r'xl/(sharedStrings|worksheets/sheet\d+)\.xml', ('</si>', '</row>', '</c>'))
-    wb = load_workbook(str(path), read_only=True, data_only=True)
-    parts = []
-    for ws in wb.worksheets:
-        parts.append(f'# sheet: {ws.title}')
-        for row in ws.iter_rows(values_only=True):
-            cells = [str(c) for c in row if c is not None]
-            if cells:
-                parts.append('\t'.join(cells))
-    return '\n'.join(parts)
+    # Shared strings hold most cell text; sheet XML holds the rest.
+    return _ooxml_text(path, r'xl/(sharedStrings|worksheets/sheet\d+)\.xml', ('</si>', '</row>', '</c>'))
 
 
 def gateway_key() -> str:

indexer/reindex.sh

@@ -34,6 +34,18 @@ fi
 PYTHON_BIN="${PYTHON_BIN:-python3}"
 echo "python: $PYTHON_BIN ($($PYTHON_BIN --version 2>&1))"
 
+# Office documents convert to Markdown through downmark, bundled with the
+# server, before enrichment; enrich.py reuses those cache entries and keeps
+# its own readers for anything that pass skipped. Needs the built server.
+PREEXTRACT="$PROJECT_ROOT/server/dist/preextract.js"
+NODE_BIN="${NODE_BIN:-node}"
+if [ -f "$PREEXTRACT" ] && command -v "$NODE_BIN" >/dev/null 2>&1; then
+  "$NODE_BIN" "$PREEXTRACT" --kb-root "$KB_ROOT" --extract-cache "$INDEX_BASE/extract" \
+    || echo "WARNING: office pre-extraction failed; enrich.py uses its own readers"
+else
+  echo "server/dist/preextract.js or node not found; enrich.py uses its own office readers"
+fi
+
 "$PYTHON_BIN" "$PROJECT_ROOT/indexer/enrich.py" --kb-root "$KB_ROOT" \
   --index "$STAGING/$(basename "$KB_ROOT")" --extract-cache "$INDEX_BASE/extract"
 

indexer/requirements.txt

@@ -1,4 +1,2 @@
-python-docx
+# Optional: PDF text fallback for hosts without poppler (pdftotext).
 pypdf
-python-pptx
-openpyxl

server/package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@fastify/multipart": "^10.1.0",
     "@fastify/static": "^8.0.0",
+    "@giraffesyo/downmark": "0.6.0",
     "@parallelworks/workflow-parser": "^0.1.0",
     "fastify": "^5.0.0",
     "jose": "^6.2.8",

server/src/config.ts

@@ -81,7 +81,7 @@ export const TEXT_SUFFIXES = new Set([
   '.html', '.css', '.js', '.ts', '.tsx', '.svg', '.toml', '.cfg', '.ini', '.def', '.mjs', '.sql',
   '.log', '.conf', '.env', '.service',
 ])
-export const EXTRACTED_SUFFIXES = new Set(['.pdf', '.docx', '.pptx', '.xlsx'])
+export const EXTRACTED_SUFFIXES = new Set(['.pdf', '.docx', '.pptx', '.xlsx', '.doc'])
 export const IMAGE_SUFFIXES = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp'])
 export const MODEL_SUFFIXES = new Set(['.stl', '.step', '.stp'])
 

server/src/indexing.ts

@@ -4,6 +4,7 @@ import fsp from 'node:fs/promises'
 import path from 'node:path'
 import { EXCLUDE_DIRS, GUFI_BIN, GUFI_INDEX, INDEX_BASE, KB_ROOT, PROJECT_ROOT, PYTHON_BIN } from './config.js'
 import { invalidateDbList } from './gufi.js'
+import { runPreExtract } from './preextract.js'
 import { invalidateContext } from './chat/context.js'
 import { effectiveSettings } from './settings.js'
 
@@ -44,18 +45,14 @@ let extractorCache: ExtractorReport[] | null = null
 
 export function extractorReport(): ExtractorReport[] {
   if (extractorCache) return extractorCache
-  const probe = (code: string): boolean => {
-    try { execFileSync(PYTHON_BIN, ['-c', code], { stdio: 'ignore', timeout: 15_000 }); return true }
-    catch { return false }
-  }
   const which = (bin: string): boolean => {
     try { execFileSync('sh', ['-c', `command -v ${bin}`], { stdio: 'ignore', timeout: 10_000 }); return true }
     catch { return false }
   }
+  // Office formats convert through downmark, which ships inside the server
+  // bundle, so they never depend on the host.
   extractorCache = [
-    { name: 'python-docx', available: probe('import docx'), covers: '.docx' },
-    { name: 'python-pptx', available: probe('import pptx'), covers: '.pptx' },
-    { name: 'openpyxl', available: probe('import openpyxl'), covers: '.xlsx' },
+    { name: 'downmark', available: true, covers: '.docx, .pptx, .xlsx, .doc' },
     { name: 'pdftotext', available: which('pdftotext'), covers: '.pdf' },
     { name: 'tesseract', available: which('tesseract'), covers: 'text inside images (OCR)' },
   ]
@@ -165,6 +162,8 @@ export function incrementalIndexDir(rel: string): Promise<{ ms: number }> {
       await fsp.rm(target, { recursive: true, force: true })
       await fsp.rename(built, target)
 
+      // Office documents go to Markdown first; enrich.py reuses the entries.
+      await runPreExtract({ kbRoot: KB_ROOT, cacheRoot: path.join(INDEX_BASE, 'extract'), subdir: cleaned }, m => console.warn(m))
       await run(PYTHON_BIN, [path.join(PROJECT_ROOT, 'indexer', 'enrich.py'),
         '--kb-root', KB_ROOT, '--index', GUFI_INDEX,
         '--extract-cache', path.join(INDEX_BASE, 'extract'), '--subdir', cleaned])
@@ -202,6 +201,7 @@ export function indexRootDb(): Promise<{ ms: number }> {
       const built = path.join(staging, path.basename(KB_ROOT), 'db.db')
       await fsp.mkdir(GUFI_INDEX, { recursive: true })
       await fsp.rename(built, path.join(GUFI_INDEX, 'db.db'))
+      await runPreExtract({ kbRoot: KB_ROOT, cacheRoot: path.join(INDEX_BASE, 'extract'), noRecurse: true }, m => console.warn(m))
       await run(PYTHON_BIN, [path.join(PROJECT_ROOT, 'indexer', 'enrich.py'),
         '--kb-root', KB_ROOT, '--index', GUFI_INDEX,
         '--extract-cache', path.join(INDEX_BASE, 'extract'), '--no-recurse'])