diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..a078769d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Dependabot — GitHub Actions only. +# Python deps are managed by upstream; we stay in sync via git merge. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + groups: + ci-actions: + patterns: + - "*" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f21d8b5..d619842a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,295 +1,147 @@ -name: Create Release and Publish Docker Images +# Fieldnote-Echo/Kokoro-FastAPI — Hardened GHCR Release +# Replaces upstream's multi-arch bake workflow with a GPU-only, amd64-only +# build that includes supply-chain hardening (pinned actions, Sigstore +# attestation, Trivy scan, CycloneDX SBOM). +# +# Incorporates from upstream's reworked release workflow: concurrency group, +# duplicate-release guard, ubuntu-24.04 runner, and the new multi-stage +# docker/gpu/Dockerfile.optimized build context. +# +# Triggers: tag push (v*) or manual dispatch. + +name: Release GPU Image on: push: - branches: - - release # Auto-trigger only on release branch - paths-ignore: - - '**.md' - - 'docs/**' - workflow_dispatch: # Manual trigger - explicitly specify branch - inputs: - branch_name: - description: 'Branch to build from (required)' - required: true - type: string + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: write # GitHub Release creation + packages: write # GHCR push + id-token: write # Sigstore OIDC + security-events: write # SARIF upload + attestations: write # Build provenance concurrency: - group: release-${{ github.event_name == 'workflow_dispatch' && inputs.branch_name || github.ref_name }} + group: release-${{ github.ref_name }} cancel-in-progress: false jobs: - prepare-release: + build-and-release: runs-on: ubuntu-24.04 - outputs: - version: ${{ steps.get-version.outputs.version }} - version_tag: ${{ steps.get-version.outputs.version_tag }} - source_ref: ${{ steps.resolve-ref.outputs.source_ref }} + env: + IMAGE: ghcr.io/fieldnote-echo/kokoro-fastapi-gpu steps: - - name: Resolve source ref - id: resolve-ref - run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "source_ref=${{ inputs.branch_name }}" >> $GITHUB_OUTPUT - else - echo "source_ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - fi - - - name: Checkout repository - uses: actions/checkout@v5 + # ── Runner hardening ────────────────────────────────────────────── + - name: Harden runner + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 with: - ref: ${{ steps.resolve-ref.outputs.source_ref }} - fetch-depth: 0 + egress-policy: audit - - name: Get version from VERSION file - id: get-version - run: | - VERSION_PLAIN=$(cat VERSION) - BRANCH_NAME="${{ steps.resolve-ref.outputs.source_ref }}" - - if [[ "$BRANCH_NAME" == "release" ]]; then - echo "version=${VERSION_PLAIN}" >> $GITHUB_OUTPUT - echo "version_tag=v${VERSION_PLAIN}" >> $GITHUB_OUTPUT - else - SAFE_BRANCH=$(echo "$BRANCH_NAME" | tr '/' '-' | tr '[:upper:]' '[:lower:]') - echo "version=${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT - echo "version_tag=v${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT - fi + # ── Checkout ────────────────────────────────────────────────────── + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Check tag does not exist + # ── Guard: don't re-publish an existing release ─────────────────── + - name: Check release does not exist + if: startsWith(github.ref, 'refs/tags/') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - TAG_NAME="${{ steps.get-version.outputs.version_tag }}" - git fetch --tags - if git rev-parse "$TAG_NAME" >/dev/null 2>&1; then - echo "::error::Tag $TAG_NAME already exists. Please increment the version in the VERSION file." + if gh release view "${{ github.ref_name }}" --repo "${{ github.repository }}" >/dev/null 2>&1; then + echo "::error::Release ${{ github.ref_name }} already exists. Push a new tag to publish a new release." exit 1 fi - build-images: - needs: prepare-release - permissions: - packages: write - env: - DOCKER_BUILDKIT: 1 - BUILDKIT_STEP_LOG_MAX_SIZE: 10485760 - VERSION: ${{ needs.prepare-release.outputs.version_tag }} - REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} - OWNER: ${{ vars.OWNER || 'remsky' }} - REPO: ${{ vars.REPO || 'kokoro-fastapi' }} - strategy: - matrix: - include: - - build_target: "cpu" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "gpu" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "gpu-cu128" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "cpu" - platform: "linux/arm64" - runs_on: "ubuntu-24.04-arm" - - build_target: "gpu" - platform: "linux/arm64" - runs_on: "ubuntu-24.04-arm" - - build_target: "rocm" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - runs-on: ${{ matrix.runs_on }} - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ needs.prepare-release.outputs.source_ref }} - + # ── Free disk space (CUDA images are ~12 GB) ────────────────────── - name: Free disk space run: | - echo "Listing current disk space" - df -h - echo "Cleaning up disk space..." - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc docker system prune -af - echo "Disk space after cleanup" - df -h - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - driver-opts: | - image=moby/buildkit:v0.21.1 - network=host + # ── Docker Buildx ───────────────────────────────────────────────── + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 + # ── GHCR login ──────────────────────────────────────────────────── + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push single-platform image - run: | - PLATFORM="${{ matrix.platform }}" - BUILD_TARGET="${{ matrix.build_target }}" - VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" - - echo "Building ${PLATFORM} image for ${BUILD_TARGET} version ${VERSION_TAG}" - - TARGET="${BUILD_TARGET}-$(echo ${PLATFORM} | cut -d'/' -f2)" - echo "Using bake target: $TARGET" - - # Bake reads REVISION/CREATED as HCL variables; both populate - # org.opencontainers.image.* labels and annotations on the per-arch push. - export REVISION="${GITHUB_SHA}" - export CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - docker buildx bake $TARGET --push --progress=plain \ - --set "*.cache-from=type=gha,scope=${TARGET}" \ - --set "*.cache-to=type=gha,mode=max,scope=${TARGET}" - - create-manifests: - needs: [prepare-release, build-images] - runs-on: ubuntu-24.04 - permissions: - packages: write - env: - REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} - OWNER: ${{ vars.OWNER || 'remsky' }} - REPO: ${{ vars.REPO || 'kokoro-fastapi' }} - strategy: - matrix: - build_target: ["cpu", "gpu", "gpu-cu128", "rocm"] - steps: - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 + # ── Image metadata (semver tags + latest) ───────────────────────── + - name: Extract metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create multi-platform manifest - run: | - VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" - TARGET="${{ matrix.build_target }}" - REGISTRY="${{ env.REGISTRY }}" - OWNER="${{ env.OWNER }}" - REPO="${{ env.REPO }}" - - # Resolve the package name, manifest tag(s), and source images per target. - # - # Tag layout per target: - # - cpu: :VERSION + :latest (multi-arch) - # - gpu: :VERSION + :VERSION-cu126 + :latest + :latest-cu126 - # (the -cu126 aliases make the wheel variant explicit so - # Maxwell/Pascal users can pin it across future default flips) - # - gpu-cu128: :VERSION-cu128 + :latest-cu128 (amd64 only) - # - rocm: :VERSION + :latest (amd64 only) - # - # MANIFEST_TAGS / LATEST_TAGS are space-separated lists; all entries point at - # the same SOURCES so the underlying blobs are deduplicated by digest. - # - # TITLE / DESCRIPTION are attached as index-level OCI annotations. - # GHCR reads org.opencontainers.image.description from the index manifest - # for multi-arch packages, so the per-arch labels set by bake are not - # enough on their own. - case "$TARGET" in - gpu-cu128) - PACKAGE="${REPO}-gpu" - MANIFEST_TAGS="${VERSION_TAG}-cu128" - LATEST_TAGS="latest-cu128" - SOURCES=("${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu128-amd64") - TITLE="Kokoro-FastAPI (GPU, cu128 / Blackwell)" - DESCRIPTION="Kokoro TTS served via FastAPI. NVIDIA GPU build with CUDA 12.8 + cu128 torch wheels for RTX 50-series (Blackwell, sm_120). amd64 only." - ;; - gpu) - PACKAGE="${REPO}-gpu" - MANIFEST_TAGS="${VERSION_TAG} ${VERSION_TAG}-cu126" - LATEST_TAGS="latest latest-cu126" - # Per-arch sources carry their wheel variant (cu126 amd64, cu129 arm64); - # both feed the same multi-arch manifest tags above. - SOURCES=( - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu126-amd64" - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu129-arm64" - ) - TITLE="Kokoro-FastAPI (GPU)" - DESCRIPTION="Kokoro TTS served via FastAPI. NVIDIA GPU build, multi-arch (CUDA 12.6 + cu126 wheels on amd64, CUDA 12.9 + cu129 wheels on arm64). See the -cu128 tag for Blackwell." - ;; - rocm) - PACKAGE="${REPO}-rocm" - MANIFEST_TAGS="${VERSION_TAG}" - LATEST_TAGS="latest" - SOURCES=("${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-amd64") - TITLE="Kokoro-FastAPI (ROCm)" - DESCRIPTION="Kokoro TTS served via FastAPI. AMD ROCm build. amd64 only." - ;; - *) - PACKAGE="${REPO}-${TARGET}" - MANIFEST_TAGS="${VERSION_TAG}" - LATEST_TAGS="latest" - SOURCES=( - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-amd64" - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-arm64" - ) - TITLE="Kokoro-FastAPI (CPU)" - DESCRIPTION="Kokoro TTS served via FastAPI. CPU build, multi-arch (amd64 + arm64)." - ;; - esac - - # Common index annotations. `index:` is required so these land on the - # index manifest itself, not on per-platform descriptors inside it. - SOURCE_URL="https://github.com/${OWNER}/Kokoro-FastAPI" - CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - REVISION="${GITHUB_SHA}" - ANNOTATIONS=( - --annotation "index:org.opencontainers.image.title=${TITLE}" - --annotation "index:org.opencontainers.image.description=${DESCRIPTION}" - --annotation "index:org.opencontainers.image.source=${SOURCE_URL}" - --annotation "index:org.opencontainers.image.url=${SOURCE_URL}" - --annotation "index:org.opencontainers.image.licenses=Apache-2.0" - --annotation "index:org.opencontainers.image.version=${VERSION_TAG}" - --annotation "index:org.opencontainers.image.revision=${REVISION}" - --annotation "index:org.opencontainers.image.created=${CREATED}" - ) + images: ${{ env.IMAGE }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix= + + # ── Build & push ────────────────────────────────────────────────── + # CUDA base images are pinned by digest, mirroring docker-bake.hcl's + # gpu-amd64 target (CUDA 12.6.3). Keep these in sync with the bake file; + # refresh with: docker buildx imagetools inspect nvcr.io/nvidia/cuda: + - name: Build and push + id: build + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + file: docker/gpu/Dockerfile.optimized + platforms: linux/amd64 + push: true + build-args: | + CUDA_VERSION=12.6.3 + CUDA_BUILDER_IMAGE=nvcr.io/nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04@sha256:50efab398f76258daa91ceebb33b6467e40217c67ea44fb5a2cebc6be7d9cce3 + CUDA_RUNTIME_IMAGE=nvcr.io/nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04@sha256:8aef630a54bc5c5146ae5ce68e6af5caa3df0fb690bb91544175c91f307e4356 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ── Sigstore build provenance ───────────────────────────────────── + - name: Attest build provenance + uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true - for TAG in $MANIFEST_TAGS; do - docker buildx imagetools create \ - "${ANNOTATIONS[@]}" \ - -t ${REGISTRY}/${OWNER}/${PACKAGE}:${TAG} \ - "${SOURCES[@]}" - done + # ── Trivy container scan ────────────────────────────────────────── + - name: Trivy vulnerability scan + uses: aquasecurity/trivy-action@c1824fd6edce30d7ab345a9989de00bbd46ef284 # 0.34.0 + with: + image-ref: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }} + format: sarif + output: trivy-results.sarif + severity: CRITICAL,HIGH - # Publish the rolling tag(s) only for clean release tags (no branch suffix). - if [[ "$VERSION_TAG" != *"-"* ]]; then - for TAG in $LATEST_TAGS; do - docker buildx imagetools create \ - "${ANNOTATIONS[@]}" \ - -t ${REGISTRY}/${OWNER}/${PACKAGE}:${TAG} \ - "${SOURCES[@]}" - done - fi + - name: Upload Trivy SARIF + uses: github/codeql-action/upload-sarif@9e907b5e64f6b83e7804b09294d44122997950d6 # v4.32.3 + with: + sarif_file: trivy-results.sarif - create-release: - needs: [prepare-release, create-manifests] - runs-on: ubuntu-24.04 - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v5 + # ── CycloneDX SBOM ─────────────────────────────────────────────── + - name: Generate SBOM + uses: anchore/sbom-action@28d71544de8eaf1b958d335707167c5f783590ad # v0.22.2 with: - fetch-depth: 0 + image: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }} + format: cyclonedx-json + output-file: sbom.cdx.json + # ── GitHub Release ──────────────────────────────────────────────── - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.prepare-release.outputs.version_tag }} - target_commitish: ${{ needs.prepare-release.outputs.source_ref }} - name: Release ${{ needs.prepare-release.outputs.version_tag }} - body: | - Curated summary: [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/${{ needs.prepare-release.outputs.version_tag }}/CHANGELOG.md) - generate_release_notes: true - draft: false - prerelease: false + if: startsWith(github.ref, 'refs/tags/') env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${{ github.ref_name }}" \ + --title "Release ${{ github.ref_name }}" \ + --generate-notes \ + sbom.cdx.json diff --git a/NARRATION_STRATEGY.md b/NARRATION_STRATEGY.md new file mode 100644 index 00000000..b11f975a --- /dev/null +++ b/NARRATION_STRATEGY.md @@ -0,0 +1,158 @@ +# Decision Brief: Kokoro-Based Natural Narration (and the Singing Horizon) + +*Synthesized from 5 research angles + adversarial verification. Where a verifier corrected a finding, the corrected version is what appears below — several load-bearing claims changed under scrutiny, flagged inline as **[verified]** or **[corrected]**.** + +--- + +## 1. HOW AI SINGING IS DONE — Taxonomy & Where We Sit + +There are three distinct families. They differ primarily by **what you feed in at inference time**. + +| Family | Input at inference | Training data needed | Quality ceiling | Examples | +|---|---|---|---|---| +| **SVS (score-trained)** | Symbolic music score (phoneme + pitch-ID + note-duration streams) + lyrics. **No reference audio.** | Paired score/singing corpus per voice | Highest for singing; still fighting for every 0.5 MOS | DiffSinger, VISinger2, NNSVS; commercial: Synthesizer V, ACE Studio | +| **SVC (voice conversion)** | An **existing sung performance (audio)** + target-speaker embedding. Never sees a score. | ~10 min target-voice audio | High; preserves source expressivity | RVC (MIT), so-vits-svc (AGPL-3.0) | +| **Speech-to-singing / vocoder-retune (OURS)** | A **spoken utterance** + a target melody (note pitches/durations). DSP F0 substitution. | **Zero.** No neural training at all. | Lowest of the three — structural ceiling | WORLD/STRAIGHT pipelines; Saitou & Goto 2007 | + +**Key architectural facts (verified):** +- SVS systems (DiffSinger, VISinger2, NNSVS) genuinely take a symbolic score and need **no reference audio at inference** — this is the clean distinction from SVC. **[corrected]** Caveat: their singer *timbre* is still learned from training-time audio, and a 2025-26 zero-shot SVS branch (e.g. "Everyone-Can-Sing") *reintroduces* an inference-time reference clip for timbre cloning. So "no reference audio" is a property of *classic* score-driven SVS, not the category's future. +- DiffSinger's shallow-diffusion trick (start reverse diffusion partway using a cheap aux decoder's mel prediction, not pure noise) is **[verified]** exactly as described; openvpi fork is Apache-2.0, 44.1kHz, swappable vocoders. +- **[corrected]** NNSVS's autoregressive F0 model is framed by its own paper as an *alternative to* explicit vibrato modeling ("can generate a more natural voice without explicitly modeling vibrato"), **not** a dedicated vibrato mechanism. Minor, but don't cite it as "the vibrato model." +- RVC/so-vits-svc pipeline (content encoder → F0 extractor (RMVPE) → VITS → NSF-HiFiGAN) is **[verified]**. License gap is real and matters: **RVC is MIT, so-vits-svc is AGPL-3.0** (network-use source-disclosure obligation). + +### Where OUR speech-to-singing pipeline sits — and its honest ceiling + +Our recipe (TTS word-timestamps as forced alignment → WORLD F0-flatten-to-notes + vibrato → frame-stretch to note durations, zero training) is a **modern instance of the Saitou & Goto 2007 STRAIGHT approach**, now on WORLD. That lineage is real and works. + +**The ceiling is structural, not a tuning problem [verified]:** +- **V/UV boundary errors:** WORLD estimates F0 (DIO), spectral envelope (CheapTrick), aperiodicity (D4C) as *independent* streams. They can disagree about which frames are voiced at exactly the note onsets/offsets a singing pipeline cares about → audible discontinuities. +- **Formant-vs-pitch coupling:** harmonics are physically tied to F0. Forcing a new pitch onto an envelope estimated for the *old* pitch introduces the "buzzy"/distorted artifact — a 2026 paper (arXiv:2601.10345) exists purely to *repair* pitch-shift-degraded singing, i.e. this is treated as an unsolved problem. +- **Regime limits:** STS quality degrades on notes >150ms and beyond the speaker's corpus vocal range — exactly the sustained-note, wide-interval regime narration-to-singing hits. +- **Trained SVS itself tops out ~2.85–4.0 MOS** (ground-truth-vocoded ceiling ~4.04). A retune-of-speech approach sits *below even a naive trained SVS baseline*. Expect meaningfully lower, especially on sustained notes and vibrato. + +**[corrected] One myth to drop:** the "singing F0 range ~80–3400 Hz" figure is wrong — it conflates fundamental frequency (realistically ~65–1500 Hz) with the **singing formant** (a ~2–3.4 kHz spectral resonance). Related **[corrected]** nuance: DSP retuning *can* engineer singing-specific timbre cues — Saitou 2007 explicitly hand-builds a singer's-formant boost and vibrato-synced formant AM, and rated near-real-singing in listening tests. So the honest framing is not "DSP *cannot* model singing timbre" but "DSP models it via **hand-engineered rules**, not **learned** from data — which caps flexibility on subtler cues (breathiness, register transitions, singer idiosyncrasy)." + +**Bottom line:** Our approach's *only* durable advantage is **zero singing-training-data + any TTS voice retunable**. It is the right tool for a demo/novelty/horizon feature, the wrong tool to compete on singing quality. Treat it exactly as the concept already does: **horizon, not deliverable.** + +--- + +## 2. OUR CONCEPT: BETTER / WORSE / HOPEFUL + +### Tier 1 — md→phoneme/prosody usability layer (the NEAR-TERM GOAL) + +**This is the right bet, but it is low-novelty — that's fine, novelty isn't the goal, natural LMS narration is.** + +- **WORSE / already solved by others:** The pattern of "have the upstream LLM emit inline IPA + emphasis markup before text hits the TTS" is **already shipped vendor practice** — Inworld AI's docs literally instruct developers to prompt their LLM to wrap emphasis in asterisks and replace proper nouns with IPA. ElevenLabs v3 ships bracketed Audio Tags for the same seam. LLM-generated SSML tag placement for narration is a *published, quantified* result (ICNLSP 2025: ~50% listener-preference lift on French audiobooks with Qwen). So we are not inventing a category. +- **BETTER (our actual edge):** every one of those is **closed** and solves pronunciation as a **post-hoc, per-word patch** (pronunciation dictionaries, SSML ``, phonetic respelling) **disconnected from the content-generation step**. Articulate Storyline 360 — the dominant authoring tool — *has no pronunciation dictionary at all*, forcing authors to misspell words phonetically. The whitespace: **the LLM that drafts the course script emits the phoneme/prosody markup natively, as part of drafting**, into a phoneme-native open model we control end-to-end. Nobody ships that closed loop. +- **HOPEFUL but honest — scale is partly eating this:** Amazon's BASE TTS shows that at 10K+ training hours, "emergent" prosody appears and models infer correct *contextual/segmental* prosody from raw text with **no markup**. So the "not robotic" problem is *partly* solved by just using a good model. BUT the same paper shows scale does **not** solve emotional/emphasis intent (stuck ~2.0/3 even at max scale) or paralinguistics without explicit keywords. **Implication:** for plain "read this clearly and naturally," lean on the model + light normalization; reserve explicit markup for the things scale demonstrably *doesn't* fix — jargon pronunciation, acronyms, numbers, deliberate emphasis. Don't over-engineer prosody markup where a good voice already handles it. + +### Tier 2 — phoneme-native LLM output (the BET / HORIZON) + +**Mechanism is real and researched; the specific "external LLM hand-authors a score a frozen TTS honors" variant is unproven — and Kokoro specifically is a weak substrate for it.** + +- **BETTER / real precedent:** The decomposition "LM predicts explicit prosody tokens *first*, then conditions speech generation" is proven to help (RALL-E: WER 6.3%→2.8%; ProsodyLM; BatonVoice's "conductor LLM emits textual vocal-feature plan, BatonTTS performs it" is architecturally *our exact shape*). DiffSinger's phoneme+pitch-ID+note-duration input **is** the "explicit prosody score" interface Tier-2 wants — proving our concept rediscovers an established pattern rather than inventing one. +- **WORSE / the falsifying reality:** In *every* published case the prosody tokens are predicted by **the TTS system's own internal LM**, not authored by an upstream general-purpose LLM and honored by a frozen TTS. **Nobody has shown external-authored prosody tokens honored by a frozen model.** And the field's momentum is the *opposite direction* — natural-language style prompts (Parler-TTS, PromptTTS2, InstructTTS), i.e. describe prosody in English prose, not phonetic markup. +- **Kokoro-specific constraints (heavily revised under verification):** + - Phoneme injection `[word](/ipa/)` **works [verified]** — but **breaks on very large inputs** (~40k chars: model speaks *both* the word and the phonetic hint). **Mitigation: chunk text.** This is a real operational constraint, not a blocker. + - **[corrected — important]** The earlier "stress markup is non-functional" finding is **wrong**. Issue #170 is *closed*; the maintainer confirmed stress changes work on ≥0.9.4, and a second user confirmed the effect **does occur but is subtle and sentence-dependent** ("don't expect miracles"). So Kokoro *does* respond to `[word](+1/+2/-1/-2)` stress — just weakly. This is a *usable* signal for light emphasis, not a dead end. + - **[corrected]** The "SSML request unanswered" finding is **refuted** — issue #36 got a maintainer reply (2025-02-04) pointing to markdown pronunciation syntax + Lexicon overrides as the sanctioned surfaces, and noting emotion is blocked by **training data** (never trained on sad/angry), not by missing SSML parsing. Takeaway: emotion control won't come from markup on Kokoro at all. + - **[verified]** `speed` is `Union[float, Callable[[int],float]]` — a callable gives **per-segment** rate control (not just one global float). Under-appreciated lever. + - **[verified]** Kokoro inherits StyleTTS2's internal per-phoneme ProsodyPredictor (duration/F0/energy) but it is **not exposed** as a settable input. FastSpeech2 is the cleanest open model that *does* expose inference-time pitch/energy/duration control — but only as **scalar multipliers**, not arbitrary contours. If we ever want true per-phoneme control, forking Kokoro to expose the ProsodyPredictor (FastSpeech2-style) is the architectural template. + +**Tier-2 verdict:** Genuinely interesting, genuinely unproven, and Kokoro is not the model that will validate it (StyleTTS2's learned diffusion prosody prior may *fight* injected scores rather than compose with them — an open empirical question). Keep Tier-2 as a **research spike behind the Tier-1 product**, not a roadmap commitment. What reliably works as markup across modern open TTS is **discrete trained-in tags** ([laugh], [S1]) — Orpheus/CSM/Dia — not continuous prosody values. That tells us the honorable path is discrete, trained-in vocabulary, not free-form pitch curves bolted onto a frozen model. + +--- + +## 3. MARKET NICHE — Sharpest Defensible Position + +**The niche: "Correctness-first, self-hostable course narration where the script generator and the voice speak the same phonetic language."** + +Four combined properties none of the incumbents hold together: +1. **Open-source + Apache-2.0** (Kokoro) — self-hostable, no per-minute metering, data stays in-house. +2. **Phoneme-controllable** with an explicit IPA injection port. +3. **LLM-integrated at generation time** (markup authored *during* drafting, not patched after). +4. **LMS-embedded** — sits inside the course-authoring pipeline, not a separate voiceover SaaS. + +**Who we beat, and on what axis:** + +| Competitor | We beat them on | Their advantage over us | +|---|---|---| +| **ElevenLabs / Murf / WellSaid / PlayHT** | Cost at scale (they meter $0.15–0.30/finished-min; we self-host at ~zero marginal), data residency, generation-time pronunciation vs. their post-hoc per-word patching | Raw naturalness, voice cloning, polish | +| **Synthesia** | Price (~$2.90–3.00/min — video-bottlenecked), pure-audio focus | Talking-head video (different lane) | +| **Articulate Storyline 360 / Adobe Captivate (Polly/Azure behind them)** | They have *no* (Storyline) or legacy (Captivate) pronunciation dictionary; we make jargon/acronym correctness a first-class generation feature | Incumbent LMS integration, install base | +| **Piper / Coqui XTTS (open alternatives)** | Piper pivoted to **GPL-3.0** (Oct 2025) and has no phoneme injection; Coqui's good model (XTTS-v2) is **non-commercial** weights and dropped espeak-ng | Coqui has voice cloning | + +**The defensible axis is CORRECTNESS + CONTROL + COST, not naturalness.** We will lose a blind naturalness A/B to ElevenLabs. We win on: *"technical training content where mispronounced jargon/acronyms/numbers is unacceptable, generated at LMS scale without per-minute fees, self-hosted."* Kokoro's training distribution is **long-form reading/narration [verified]** — already the right style for course VO, a poor fit for conversational competitors. Market is large and growing (AI-voice ~$5.6–7.7B in 2026 → $21–33B by 2030-32; every vendor ships a pronunciation-editor UI, proving demand) but **no one lets the content-generation step emit the markup**. That is the wedge. + +**Honest caveat:** "correctness-first" is a real but *narrow* wedge — it's a feature, not yet a moat. The moat is the closed loop (LLM + phoneme model + LMS) being *ours end-to-end* and *open*. + +--- + +## 4. BUILD PRIMITIVES & LESSONS — Actionable Modules + +### 4.1 Core data structure: the Prosody-Annotated Phoneme IR + +A single intermediate representation everything else operates on. Proposed shape — a list of tokens: + +``` +Token { + text: str # original grapheme span + phonemes: str|None # IPA override (None = let misaki G2P handle it) + stress: int # -2..+2, maps to Kokoro [word](+N) — WORKS but subtle + emphasis: bool # drives punctuation/respelling tactics, not a fake tag + break_after_ms: int # realized via punctuation/ellipsis, not SSML + source: enum # {g2p_default, lexicon, llm_authored, human_override} +} +``` +`source` provenance is load-bearing — it lets the eval harness attribute regressions and lets human overrides always win. This IR is the contract between the LLM normalizer, the lexicon, and the Kokoro adapter. + +### 4.2 Pipeline stages (each a swappable module) + +1. **Markdown/structure normalizer** — strip/interpret markdown, expand lists/headings into speakable prose, handle code blocks (spell or skip). +2. **Number/acronym/unit expander** — deterministic, rule-based FIRST (dates, currency, units, "API"→"A P I" vs "NASA"→"nassa"). This is where scale-based TTS *fails* (BASE TTS: acronyms read letter-by-letter), so it's high-value and cheap. +3. **Lexicon resolver** — see 4.4. +4. **LLM prosody/IPA annotator** — fills `phonemes`/`stress`/`emphasis` for spans not covered by lexicon. Must be prompt-engineered + post-validated (see lesson below). +5. **Kokoro adapter** — emits `[word](/ipa/)` + `[word](+N)` + punctuation, **chunks to <~40k chars**, uses the `speed` callable for per-segment pacing. +6. **(Horizon) WORLD retune module** — separate, off the critical path. + +### 4.3 Eval harness — how to A/B naturalness (build this EARLY) + +- **Objective gate first:** run TTS output back through **ASR (WER)** against the intended script. Catches the failures that actually matter for e-learning — wrong pronunciations, dropped words, the injection-doubling bug. Cheap, automatable, CI-able. +- **Pronunciation unit tests:** a fixed corpus of jargon/acronym/number cases with expected phoneme or ASR-transcription targets. Regression suite for the lexicon + expander. +- **Human A/B (MOS/preference):** pairwise (our-normalized vs. raw-Kokoro vs. ElevenLabs) on real course paragraphs. Small n, forced-choice preference is more sensitive than absolute MOS. +- **[verified] Do NOT reuse speech-MOS models for any singing output** — SingMOS-Pro shows speech-MOS models fail on singing (domain gap). Singing needs its own raters. +- Instrument by IR `source` so you can say *which stage* moved the needle. + +### 4.4 Pronunciation lexicon for jargon/acronyms + +- **Format:** term → IPA (+ optional part-of-speech/context key for homographs). Store as versioned data, not code. +- **Resolution order:** human override > domain lexicon > deterministic acronym/number rules > LLM annotation > misaki G2P default. +- **[corrected] Validate LLM-authored IPA before injection.** LLM-G2P benchmark: naive prompting = **31.6% phoneme error rate**; optimized prompting + dictionary post-processing brought Claude 3.5 Sonnet to **5.8% PER**. Dominant failure is **inconsistent symbol choice for the same sound** (silent corruption), not obvious hallucination. So: validate LLM IPA against **Kokoro/misaki's actual phoneme inventory** and reject/normalize off-vocabulary symbols. Compounding-error risk: espeak-ng (Kokoro's fallback) itself mishandles homographs and foreign words, and **there's no downstream repair once bad phonemes enter the TTS.** +- **[corrected] Consider "LLM improves the G2P, not LLM in the inference loop."** "Fast, Not Fancy" (2025) argues augmenting fast rule-based G2P with LLM-*generated training data* beats calling an LLM at inference. Cheaper, lower-latency, deterministic at runtime — a strong option for the lexicon-building phase. + +### 4.5 Lessons already learned (bank these) + +- **[corrected] Kokoro responds to stress markup — but subtly.** `[word](+1..+2/-1..-2)` works (issue #170 closed, maintainer + user confirmed) on short/less-stressed words, effect is subtle and sentence-dependent. Usable for light emphasis; don't expect strong control. +- **[corrected] Kokoro does NOT do emotion via markup — ever.** Blocked by training data (never trained on emotion categories), per maintainer. Don't build emotion control on Kokoro; it needs a different model or fine-tune. +- **[verified] Phoneme injection is reliable at normal input sizes, breaks ~40k chars → chunk.** +- **[verified] `speed` accepts a callable for per-segment pacing** — free pacing control, currently unused. +- **Punctuation, ellipses, and phonetic respelling are the *reliable* prosody levers** across neural TTS; SSML-style tags are widely ignored. Prefer orthographic tactics over markup where possible. +- **[verified] WORLD retune works but has a hard, structural ceiling** (V/UV boundary errors, formant-pitch coupling, degrades >150ms notes / beyond corpus range). Horizon feature only. And drop the bogus "80–3400Hz F0" figure. +- **Forced alignment is free** — Kokoro's word-level timestamps *are* the alignment; no aligner needed for the retune pipeline. +- **Discrete trained-in tags are the only markup that reliably works** in modern open TTS (Orpheus/CSM/Dia). If we ever want robust expressive control, the path is trained-in vocabulary, not free-form continuous markup on a frozen model. + +--- + +## 5. RECOMMENDED FIRST MOVES (ordered) + +1. **Build the ASR-based eval harness + pronunciation unit-test corpus first.** You cannot claim "less robotic" or "correct" without measurement, and this catches the injection-doubling and jargon failures immediately. Everything downstream is graded against it. *(Smallest, highest-leverage, unblocks all A/B claims.)* + +2. **Ship the deterministic normalizer + jargon/acronym/number lexicon** (pipeline stages 1–3, IR from 4.1). This is where scale-based TTS provably fails and where our correctness wedge lives — rule-based, no LLM/model risk, immediately demoable on real Oxford House course scripts. + +3. **Add the LLM IPA/prosody annotator with a validation gate** (stage 4 + 4.4). Prompt-engineer + post-validate against Kokoro's phoneme inventory; measure PER against the corpus from step 1. Gate: must beat raw-Kokoro on the eval before it ships. Keep it *narrow* — jargon/emphasis where scale doesn't help — rather than annotating everything. + +4. **Run the first blind A/B** (our-normalized vs. raw-Kokoro vs. one commercial baseline) on real course paragraphs. Decide with data whether the Tier-1 layer actually moves naturalness/correctness enough to build the product around, and *which stage* did the work. + +5. **Time-box a Tier-2 spike (research, not product):** test whether Kokoro *honors* externally-authored stress/prosody markup at all beyond the known-subtle stress effect — one afternoon, tight hypothesis, using the harness. Result feeds the "is the phoneme-native bet real on this model?" question without committing roadmap. Keep the WORLD singing pipeline entirely off the critical path as the demo/horizon artifact. + +**Guiding principle:** the near-term goal (natural, *correct* LMS narration) is achievable, defensible, and mostly *engineering*, not research. The horizon (phoneme-native LLM output, singing) is genuinely interesting but unproven and — for singing especially — capped by physics on our chosen approach. Fund the goal; spike the horizon; don't confuse them. \ No newline at end of file diff --git a/NORTH_STAR.md b/NORTH_STAR.md new file mode 100644 index 00000000..97cec11b --- /dev/null +++ b/NORTH_STAR.md @@ -0,0 +1,98 @@ +# North Star: Correct, Open, Tunable Narration + +*Distilled from the AI-singing/TTS landscape research (see `NARRATION_STRATEGY.md`). +This supersedes "singing" as the goal. Singing was the demo that taught us where +the real gap is; it stays a horizon toy, off the critical path.* + +--- + +## The one sentence + +**A pronunciation-and-prosody control layer that lets an author — or the LLM +drafting the script — guarantee *how* an open TTS says something, not just hope +it guesses right; correct by default, tunable when it isn't, and open enough to +give back.** + +## What it is + +- **Correct.** Jargon, acronyms, numbers, units, and names come out right the + first time. Wrong pronunciation in training content isn't a polish issue — it's + a correctness bug, and we treat it like one (tested, gated, regressible). +- **Tunable.** When the default is wrong, the author fixes it *once*, in a + versioned lexicon, in the same phonetic language (IPA) the model already speaks + through its G2P — not by misspelling words to trick the voice. +- **Open.** Built on Apache-2.0 Kokoro, self-hostable, no per-minute meter, data + stays in-house. And the correctness layer itself is contributed back, not hoarded. + +## What it is NOT + +- **Not singing.** Vocoder-retune singing is physics-capped (V/UV boundary errors, + formant–pitch coupling, degrades past ~150ms notes). Fun demo, permanent horizon. +- **Not a naturalness play.** We will lose a blind naturalness A/B to ElevenLabs + and that's fine. Modern large TTS already infers good prosody from raw text; we + don't out-scale them and won't try. +- **Not markup-for-its-own-sake.** Reserve explicit control for what scale + provably *fails* at — pronunciation of jargon/acronyms/numbers and deliberate + emphasis. Everything else, let the model do its job. + +## Why this is the right long-horizon goal + +The market wedge that survived adversarial scrutiny: **correctness + control + +cost, not naturalness.** Incumbents (ElevenLabs, Murf, WellSaid) patch +pronunciation *post-hoc, per word,* behind closed metered APIs. The dominant +authoring tool (Articulate Storyline) has **no pronunciation dictionary at all.** +No one lets the content-generation step emit the pronunciation/prosody markup as +part of drafting. That closed loop — LLM + phoneme-native open model + authoring +pipeline, owned end-to-end and open — is the defensible position, and it's most +valuable precisely for technical training content where a mispronounced acronym +is unacceptable. + +## The good-faith open contribution + +Kokoro today has no lexicon / pronunciation-control layer. The parts of our work +that *every* Kokoro user benefits from should go **upstream** (to Kokoro-FastAPI +or the misaki G2P project), not stay in our fork: + +| Contribute upstream (good-faith) | Keep as our product | +|---|---| +| The Prosody-Annotated Phoneme IR (a clean token contract) | LMS/authoring integration (rippling-lms) | +| A pronunciation **lexicon resolver** + versioned lexicon format | Our domain lexicon (Oxford House jargon) | +| LLM→IPA **validation gate** (reject off-inventory phonemes) | The script-generation prompt chain | +| ReDoS/normalizer hardening *(already PR'd: remsky#489)* | The closed generation→narration loop | + +The ReDoS fix we already sent upstream is the template: find a real gap in a +solid project, fix it cleanly and in isolation, give it back. The pronunciation +layer is the same move at larger scale — it makes Kokoro better for everyone and +makes our product possible at the same time. + +## The shape of the contribution (concrete) + +A **Prosody-Annotated Phoneme IR** as the contract between stages: + +``` +Token { text, phonemes|None, stress:-2..+2, emphasis:bool, + break_after_ms:int, source: {g2p_default|lexicon|llm_authored|human_override} } +``` + +Pipeline (each stage swappable, upstreamable pieces marked *): +1. Structure/markdown normalizer ← our md work, already built & hardened +2. Number/acronym/unit expander* (rule-based; where big TTS fails) +3. Lexicon resolver* (human > domain > rules > LLM > G2P default) +4. LLM IPA/prosody annotator + validation gate* (naive LLM→IPA is 31.6% PER; + validated against Kokoro's phoneme inventory → ~5.8%) +5. Kokoro adapter (emit `[word](/ipa/)` + `[word](+N)`, chunk <40k, `speed` callable) + +Graded by an **ASR-based eval harness** (WER against intended script) plus a +pronunciation unit-test corpus — build this first; it's what lets us *prove* +"correct," not just claim it. + +## Hard-won constraints (don't relearn these) + +- Kokoro stress markup `[word](+N)` works but is **subtle**; emotion markup + **never** works (not in its training data) — don't build emotion on Kokoro. +- **Validate every LLM-authored IPA** against the phoneme inventory before + injection; the failure mode is silent symbol inconsistency, not obvious garbage. +- Punctuation / respelling are the *reliable* prosody levers; SSML-style tags are + widely ignored by modern open TTS. +- Phoneme injection breaks ~40k chars → chunk. Word-level timestamps are free + forced alignment. `speed` takes a callable (unused pacing lever). diff --git a/README.md b/README.md index e46a04b5..fd6b6ca8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,19 @@ +> **Fieldnote-Echo fork** of [remsky/Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) (currently based on `v0.6.0`) +> +> **Goal: correct, open, tunable narration.** Kokoro can *say* words but can't yet +> *guarantee* it says them correctly, and no open TTS gives authors real control over +> pronunciation. This fork is building that layer — correct-by-default pronunciation +> and prosody, tunable when it isn't — aimed at technical/e-learning narration. Not a +> naturalness contest, not singing. See [`ROADMAP.md`](./ROADMAP.md) and [`NORTH_STAR.md`](./NORTH_STAR.md). +> +> **Shipping today:** +> - **Markdown-aware TTS normalization** — strips `**bold**`, `[links](url)`, headings, code blocks, tables, etc. before synthesis so LLM output sounds natural (toggle: `markdown_normalization` in `NormalizationOptions`, default on) +> - **Supply-chain hardened CI** — CUDA base images pinned by digest (`docker-bake.hcl` + release workflow), all GitHub Actions SHA-pinned, Sigstore build provenance, Trivy scan, CycloneDX SBOM per release +> - **Normalizer ReDoS hardening** — bounded the URL/email/markdown patterns that let a single request pin the event loop (also upstreamed: [remsky#489](https://github.com/remsky/Kokoro-FastAPI/pull/489)) +> - **GHCR images** — `ghcr.io/fieldnote-echo/kokoro-fastapi-gpu` (GPU, amd64) +> +> Generally-useful fixes are contributed back upstream; we rebase on `remsky/master` periodically. +

Kokoro TTS Banner

diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000..8c16c2f9 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,56 @@ +# Fork Roadmap + +*Why this fork exists, and where it's going. Full rationale in [`NORTH_STAR.md`](./NORTH_STAR.md); market/technical research in [`NARRATION_STRATEGY.md`](./NARRATION_STRATEGY.md).* + +## North star + +**Correct, open, tunable narration.** Kokoro can *say* words; it can't yet +*guarantee* it says them correctly, and no open TTS gives authors real control +over pronunciation. This fork's purpose is to close that gap — pronunciation and +prosody an author (or the LLM drafting the script) can guarantee, correct by +default and tunable when it isn't — and to contribute the generally-useful parts +back upstream. + +**Not** a naturalness contest (large TTS wins that; fine) and **not** singing +(a physics-capped demo, permanently off the critical path). The wedge is +**correctness + control + cost.** + +## Good-faith upstreaming + +Parts that help every Kokoro user go to `remsky/Kokoro-FastAPI`; the product +(LMS integration, domain lexicons, the generation→narration loop) stays here. +The ReDoS fix ([remsky#489](https://github.com/remsky/Kokoro-FastAPI/pull/489)) +is the template: find a real gap, fix it cleanly in isolation, give it back. + +## Status + +| # | Milestone | State | +|---|---|---| +| 0 | Upstream sync to v0.6.0 | ✅ done | +| 0 | Supply-chain hardening (digest pins, SHA-pinned actions, provenance, SBOM, Trivy) | ✅ done | +| 0 | Normalizer ReDoS hardening | ✅ done · upstream PR #489 | +| 1 | Markdown-aware normalization (LLM output → speakable) | ✅ done | +| 2 | **ASR-based eval harness + pronunciation test corpus** | ⏭️ next | +| 3 | Deterministic number/acronym/unit expander | planned | +| 4 | Versioned pronunciation **lexicon** + resolver | planned | +| 5 | Prosody-Annotated Phoneme **IR** (the token contract) | planned | +| 6 | LLM IPA/prosody annotator **behind a validation gate** | planned | +| 7 | Blind A/B vs raw Kokoro + one commercial baseline | planned | +| H | *(horizon)* speech-to-singing WORLD retune — demo only | spike done | + +## Milestone 2 first (why) + +You can't claim "correct" or "less robotic" without measuring it. The harness +(WER via ASR against the intended script + a jargon/acronym unit-test corpus) +grades everything downstream and instantly catches pronunciation regressions. +It's the smallest, highest-leverage piece and it unblocks every later A/B claim. + +## Constraints banked (don't relearn) + +- Kokoro stress markup `[word](+N)` works but is **subtle**; emotion markup + **never** works (not in its training data). +- **Validate LLM-authored IPA** against Kokoro's phoneme inventory before + injection — naive LLM→IPA is ~31.6% phoneme-error; validated ~5.8%. +- Punctuation/respelling are the reliable prosody levers; SSML-style tags are + widely ignored. Phoneme injection breaks ~40k chars → chunk. Word timestamps + are free forced alignment. `speed` takes a callable (unused pacing lever). diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 13d76763..95504beb 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -1,7 +1,10 @@ """ -Text normalization module for TTS processing. +Text normalization module for TTS processing — forked from remsky/Kokoro-FastAPI. Handles various text formats including URLs, emails, numbers, money, and special characters. Converts them into a format suitable for text-to-speech processing. + +Upstream: https://github.com/remsky/Kokoro-FastAPI/blob/master/api/src/services/text_processing/normalizer.py +Delta: handle_markdown() function + markdown_normalization toggle in normalize_text() """ import math @@ -155,11 +158,22 @@ MONEY_UNITS = {"$": ("dollar", "cent"), "£": ("pound", "pence"), "€": ("euro", "cent")} # Pre-compiled regex patterns for performance +# Local part and domain are bounded at their RFC limits (64 / 253 chars): +# unbounded runs made every word boundary in 'a.a.a...' floods scan the +# whole remaining string for '@' — O(n^2). EMAIL_PATTERN = re.compile( - r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-z]{2,}\b", re.IGNORECASE + r"\b[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-z]{2,}\b", re.IGNORECASE ) URL_PATTERN = re.compile( - r"(https?://|www\.|)+(localhost|[a-zA-Z0-9.-]+(\.(?:" + # Negative lookbehind: only attempt a match at token starts. Without it + # the domain branch re-scans from every position inside a long word-char + # run (O(n^2) — a 100KB run of 'a' hung the normalizer). The domain run + # is also bounded at 253 chars, the DNS name length limit. + r"(?[ \t]*)?```[^\n]*\n?(.*?)(?:```|\Z)", re.DOTALL | re.MULTILINE +) +# Blockquote marker to dedent from fenced content when the fence itself was +# blockquoted ('> ```' ... '> code' ... '> ```'). +_MD_BLOCKQUOTE_DEDENT = re.compile(r"^>[ \t]?", re.MULTILINE) +# Placeholder restore patterns (Phase 3) — input NULs are stripped up front, +# so these can only match placeholders this module minted. +_MD_CB_PLACEHOLDER = re.compile("\x00CB(\\d+)\x00") +_MD_IC_PLACEHOLDER = re.compile("\x00IC(\\d+)\x00") +_MD_INLINE_CODE = re.compile(r"`([^`]+)`") +# URL part tolerates one level of balanced parens, e.g. wiki/Foo_(bar) — +# enough for real-world URLs without over-matching past the closing ). +# Link text and URL are length-bounded like emphasis: on '[' floods the +# unbounded '[^\]]*' scanned to end-of-string from every position (O(n^2)). +_MD_URL_PART = r"(?:[^()]|\([^()]*\)){1,2000}" +_MD_IMAGE = re.compile(r"!\[([^\]]{0,800})\]\(" + _MD_URL_PART + r"\)") +# Link text may be empty ('[](url)') — the whole link then drops entirely. +_MD_LINK = re.compile(r"\[([^\]]{0,800})\]\(" + _MD_URL_PART + r"\)") +# Emphasis content: spans single newlines (soft wraps) but never a blank +# line (paragraph break) — CommonMark-ish. Bounded at 600 chars so a flood +# of unclosed markers scans O(n·600) instead of O(n^2); longer "spans" are +# not real emphasis and simply stay unstripped. +_MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n)){1,600}?" +_MD_BOLD = re.compile(r"\*\*(" + _MD_EMPHASIS_CONTENT + r")\*\*") +# Bold with __ — word-boundary-aware like the underscore italic below. +_MD_BOLD_UNDER = re.compile(r"(? 'H') — but '#' glued to a word stays ('C#'). +_MD_HEADING = re.compile( + r"^[ \t]*#{1,6}[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$", re.MULTILINE +) +_MD_BLOCKQUOTE = re.compile(r"^>\s?", re.MULTILINE) +_MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) +# List indentation/spacing is spaces and tabs only — '\s' would let the +# MULTILINE '^' anchor swallow every following blank line from each line +# start, which is O(n^2) on newline floods. +_MD_UNORDERED_LIST = re.compile(r"^([ \t]*)[-*+][ \t]+", re.MULTILINE) +_MD_ORDERED_LIST = re.compile(r"^([ \t]*)\d+\.[ \t]+", re.MULTILINE) +# Table separator rows like |---|---|. Detected with a single ambiguity-free +# character class plus a substring check: the old '-{3,}[\s:|-]*' form had two +# adjacent quantifiers both matching '-', giving O(n^2) backtracking on long +# dash lines. +_MD_TABLE_SEP_CHARS = re.compile(r"[ \t:|-]*") + + +def _is_table_sep_row(line: str) -> bool: + return "---" in line and _MD_TABLE_SEP_CHARS.fullmatch(line) is not None + + +def handle_markdown(text: str) -> str: + """Strip markdown formatting so TTS reads content, not syntax. + + Designed as a pre-processing pass: runs BEFORE email/URL normalization + so that markdown link syntax ``[text](url)`` is reduced to ``text`` + before the URL handler sees bare URLs. + """ + # --- Phase 1: Protect code content from markdown stripping --- + # Drop any NUL bytes first: they can't be voiced anyway, and it makes the + # \x00-delimited placeholders below collision-proof against input that + # happens to contain literal placeholder-looking text. + text = text.replace("\x00", "") + # Normalize line endings: every MULTILINE anchor and blank-line lookahead + # below assumes \n; raw \r defeats them (CRLF headings leak '##', CRLF + # blank lines stop acting as paragraph breaks for emphasis). + text = text.replace("\r\n", "\n").replace("\r", "\n") + + # Extract fenced code blocks and inline code into placeholders so that + # markdown syntax inside code (e.g. **bold**) is preserved literally. + code_blocks: list[str] = [] + + def _save_fenced(m: re.Match) -> str: + content = m.group(2) + if m.group(1): + # Fence was blockquoted — dedent the '> ' markers from content. + content = _MD_BLOCKQUOTE_DEDENT.sub("", content) + code_blocks.append(content) + return f"\x00CB{len(code_blocks) - 1}\x00" + + text = _MD_FENCED_CODE.sub(_save_fenced, text) + + inline_codes: list[str] = [] + + def _save_inline(m: re.Match) -> str: + inline_codes.append(m.group(1)) + return f"\x00IC{len(inline_codes) - 1}\x00" + + text = _MD_INLINE_CODE.sub(_save_inline, text) + + # --- Phase 2: Strip markdown formatting --- + # Images — keep alt text + text = _MD_IMAGE.sub(r"\1", text) + # Links — keep link text + text = _MD_LINK.sub(r"\1", text) + # Bold (** and __ before single-char markers to avoid partial match) + text = _MD_BOLD.sub(r"\1", text) + text = _MD_BOLD_UNDER.sub(r"\1", text) + # Italic (* and _) + text = _MD_ITALIC_STAR.sub(r"\1", text) + text = _MD_ITALIC_UNDER.sub(r"\1", text) + # Strikethrough + text = _MD_STRIKETHROUGH.sub(r"\1", text) + # List markers — strip marker, keep text. Must run before headings so + # '- # Heading' loses both the marker and the hashes. + text = _MD_UNORDERED_LIST.sub(r"\1", text) + text = _MD_ORDERED_LIST.sub(r"\1", text) + # Headings — strip leading hashes (and any closing hash sequence) + text = _MD_HEADING.sub(r"\1", text) + # Blockquotes — strip leading > + text = _MD_BLOCKQUOTE.sub("", text) + # Horizontal rules — remove entire line + text = _MD_HORIZONTAL_RULE.sub("", text) + # Tables — remove separator rows and de-pipe the table body. A real table + # is a separator row (|---|---|) plus the contiguous pipe-bearing lines + # above and below it; propagate table status outward from each separator + # row. This catches borderless multi-row and 2-column tables (which the old + # "2+ pipes AND anchored/adjacent" heuristic missed) while leaving a lone + # prose line with pipes — "either a | b | c works", no adjacent separator — + # untouched. Escaped pipes (\|) are hidden as a sentinel first so they are + # neither counted as table syntax nor de-piped, then restored at the end. + text = text.replace("\\|", "\x00EP\x00") + lines = text.split("\n") + is_sep_row = [_is_table_sep_row(line) for line in lines] + is_table_row = [False] * len(lines) + for i, sep in enumerate(is_sep_row): + if not sep: + continue + j = i - 1 + while j >= 0 and "|" in lines[j]: + is_table_row[j] = True + j -= 1 + j = i + 1 + while j < len(lines) and "|" in lines[j]: + is_table_row[j] = True + j += 1 + for i, line in enumerate(lines): + if is_sep_row[i]: + lines[i] = "" + elif is_table_row[i]: + lines[i] = line.replace("|", " ") + text = "\n".join(lines).replace("\x00EP\x00", "|") + + # --- Phase 3: Restore protected code content --- + # Single-pass sub keyed on the placeholder index: a per-item str.replace + # loop is O(placeholders x len(text)) — quadratic on fence/backtick floods. + if code_blocks: + text = _MD_CB_PLACEHOLDER.sub(lambda m: code_blocks[int(m.group(1))], text) + if inline_codes: + text = _MD_IC_PLACEHOLDER.sub(lambda m: inline_codes[int(m.group(1))], text) + + return text + + +# --------------------------------------------------------------------------- +# Upstream handlers (unchanged from v0.2.4) +# --------------------------------------------------------------------------- + INFLECT_ENGINE = inflect.engine() @@ -410,6 +595,13 @@ def handle_time(t: re.Match[str]) -> str: def normalize_text(text: str, normalization_options: NormalizationOptions) -> str: """Normalize text for TTS processing""" + # Handle markdown formatting FIRST — strips syntax like **bold**, [link](url), + # ```code```, etc. before other handlers see the raw symbols. + # Must run before email/URL normalization: markdown links contain URLs that + # should be reduced to link text, not spelled out. + if normalization_options.markdown_normalization: + text = handle_markdown(text) + # Handle email addresses first if enabled if normalization_options.email_normalization: text = EMAIL_PATTERN.sub(handle_email, text) @@ -491,8 +683,10 @@ def normalize_text(text: str, normalization_options: NormalizationOptions) -> st text = re.sub(r"(?<=\d)S", " S", text) text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) text = re.sub(r"(?<=X')S\b", "s", text) + # Bound the acronym run (real dotted acronyms are short): an unbounded + # '{2,}' backtracks O(n) from every position of an 'a.a.a...' flood — O(n^2). text = re.sub( - r"(?:[A-Za-z]\.){2,} [a-z]", lambda m: m.group().replace(".", "-"), text + r"(?:[A-Za-z]\.){2,12} [a-z]", lambda m: m.group().replace(".", "-"), text ) text = re.sub(r"(?i)(?<=[A-Z])\.(?=[A-Z])", "-", text) diff --git a/api/src/services/text_processing/text_processor.py b/api/src/services/text_processing/text_processor.py index cce7a128..727e5056 100644 --- a/api/src/services/text_processing/text_processor.py +++ b/api/src/services/text_processing/text_processor.py @@ -8,7 +8,7 @@ from ...core.config import settings from ...structures.schemas import NormalizationOptions -from .normalizer import normalize_text +from .normalizer import handle_markdown, normalize_text from .phonemizer import phonemize from .vocabulary import tokenize @@ -173,11 +173,34 @@ async def smart_split( if lang_code in ["a", "b", "en-us", "en-gb"]: processed_text = CUSTOM_PHONEMES.split(processed_text) for index in range(0, len(processed_text), 2): - processed_text[index] = normalize_text( - processed_text[index], normalization_options + segment = processed_text[index] + normalized = normalize_text(segment, normalization_options) + # Segments are rejoined with ''.join below, so any edge + # whitespace the normalizer eats would glue words onto + # the adjacent custom phoneme tokens — restore it. + if segment[:1].isspace() and not normalized[:1].isspace(): + normalized = " " + normalized + if segment[-1:].isspace() and not normalized[-1:].isspace(): + normalized = normalized + " " + processed_text[index] = normalized + + # Strip once on the final joined result, never per segment + processed_text = "".join(processed_text).strip() + elif normalization_options.markdown_normalization: + # Markdown syntax is language-independent — strip it even + # when English-specific normalization is skipped. Custom + # phoneme tokens look like markdown links, so protect them. + processed_text = CUSTOM_PHONEMES.split(processed_text) + for index in range(0, len(processed_text), 2): + processed_text[index] = handle_markdown( + processed_text[index] ) processed_text = "".join(processed_text).strip() + logger.info( + "Applied markdown normalization only; full text " + "normalization is only supported for english" + ) else: logger.info( "Skipping text normalization as it is only supported for english" diff --git a/api/src/structures/schemas.py b/api/src/structures/schemas.py index 4fd6ca0e..f7037dc0 100644 --- a/api/src/structures/schemas.py +++ b/api/src/structures/schemas.py @@ -1,4 +1,9 @@ -from email.policy import default +"""Kokoro-FastAPI schemas — forked from upstream to add markdown_normalization toggle. + +Upstream: https://github.com/remsky/Kokoro-FastAPI/blob/master/api/src/structures/schemas.py +Delta: NormalizationOptions.markdown_normalization field (default True) +""" + from enum import Enum from typing import List, Literal, Optional, Union @@ -48,6 +53,11 @@ class NormalizationOptions(BaseModel): default=True, description="Normalizes input text to make it easier for the model to say", ) + markdown_normalization: bool = Field( + default=True, + description="Strips markdown formatting for cleaner TTS output " + "(e.g., **bold** -> bold, [link](url) -> link)", + ) unit_normalization: bool = Field( default=False, description="Transforms units like 10KB to 10 kilobytes" ) diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py new file mode 100644 index 00000000..d4e5b291 --- /dev/null +++ b/api/tests/test_markdown_normalizer.py @@ -0,0 +1,566 @@ +"""Unit tests for handle_markdown() — the navi-os fork of Kokoro's normalizer.""" + +import pytest + +from api.src.services.text_processing.normalizer import handle_markdown +from api.src.services.text_processing.text_processor import smart_split +from api.src.structures.schemas import NormalizationOptions + +# ── Tests ──────────────────────────────────────────────────────────────── + + +def test_bold(): + assert handle_markdown("**bold text**") == "bold text" + + +def test_italic_star(): + assert handle_markdown("*italic*") == "italic" + + +def test_italic_underscore(): + assert handle_markdown("_italic_") == "italic" + + +def test_snake_case_preserved(): + assert handle_markdown("snake_case_var") == "snake_case_var" + + +def test_strikethrough(): + assert handle_markdown("~~deleted~~") == "deleted" + + +def test_inline_code(): + assert handle_markdown("`code`") == "code" + + +def test_fenced_code_block(): + result = handle_markdown("```python\nprint(\"hi\")\n```") + assert result.strip() == 'print("hi")' + + +def test_link(): + assert handle_markdown("[click here](https://example.com)") == "click here" + + +def test_image(): + assert handle_markdown("![alt text](image.png)") == "alt text" + + +def test_heading(): + assert handle_markdown("### My Heading") == "My Heading" + + +def test_blockquote(): + assert handle_markdown("> quoted text") == "quoted text" + + +def test_horizontal_rule(): + assert handle_markdown("---").strip() == "" + + +def test_unordered_list(): + assert handle_markdown("- item one") == "item one" + + +def test_ordered_list(): + assert handle_markdown("1. first item") == "first item" + + +def test_table(): + md = "| Name | Age |\n| --- | --- |\n| Alice | 30 |" + result = handle_markdown(md) + # Separator row removed; pipes replaced with spaces on table lines + assert "Name" in result + assert "Age" in result + assert "Alice" in result + assert "30" in result + # No pipe characters should remain + assert "|" not in result + # Separator row content should be gone + assert "---" not in result + + +def test_non_table_pipe_preserved(): + assert handle_markdown("true | false") == "true | false" + + +def test_nested_bold_italic(): + assert handle_markdown("***bold italic***") == "bold italic" + + +def test_code_block_preserves_markdown(): + """Markdown inside fenced code blocks must NOT be stripped.""" + result = handle_markdown("```\n**not stripped**\n```") + assert "**not stripped**" in result + + +def test_inline_code_preserves_markdown(): + """Markdown inside inline code must NOT be stripped.""" + result = handle_markdown("`**bold**`") + assert result.strip() == "**bold**" + + +def test_mixed_llm_output(): + """Integration test: a realistic LLM response with mixed markdown.""" + md = ( + "### Summary\n\n" + "This is **important** information about *machine learning*.\n\n" + "Key points:\n" + "- First, read the [documentation](https://docs.example.com)\n" + "- Then, run `pip install torch`\n\n" + "Here is an example:\n\n" + "```python\nmodel.train()\n```\n\n" + "> Note: this is experimental\n\n" + "| Metric | Value |\n" + "| --- | --- |\n" + "| Accuracy | 95% |\n" + ) + result = handle_markdown(md) + # Formatting artifacts should be gone + assert "**" not in result + assert "###" not in result + assert "](http" not in result + assert "| --- |" not in result + # Content should remain + assert "important" in result + assert "machine learning" in result + assert "documentation" in result + assert "pip install torch" in result + assert "model.train()" in result + assert "experimental" in result + assert "Accuracy" in result + assert "95%" in result + + +def test_multiple_code_blocks(): + """Multiple fenced code blocks should all be protected.""" + md = "```\n**block1**\n```\nSome **bold** text\n```\n*block2*\n```" + result = handle_markdown(md) + assert "**block1**" in result + assert "*block2*" in result + assert "bold" in result + assert "**bold**" not in result + + +def test_heading_levels(): + """All heading levels (1-6) should be stripped.""" + for i in range(1, 7): + hashes = "#" * i + assert handle_markdown(f"{hashes} Heading {i}") == f"Heading {i}" + + +def test_unordered_list_markers(): + """All unordered list markers (-, *, +) should be stripped.""" + assert handle_markdown("- dash item") == "dash item" + # * is also an italic marker, but at start of line with space it's a list + assert handle_markdown("+ plus item") == "plus item" + + +def test_bold_with_surrounding_text(): + assert handle_markdown("This is **bold** in context") == "This is bold in context" + + +def test_link_with_surrounding_text(): + result = handle_markdown("Click [here](https://x.com) now") + assert result == "Click here now" + + +# ── Multiline emphasis (soft wraps) ────────────────────────────────────── + + +def test_bold_across_soft_wrap(): + """Emphasis should span a single newline (soft-wrapped paragraph).""" + assert handle_markdown("**bold\ntext**") == "bold\ntext" + + +def test_italic_star_across_soft_wrap(): + assert handle_markdown("*ital\nic*") == "ital\nic" + + +def test_italic_underscore_across_soft_wrap(): + assert handle_markdown("_ital\nic_") == "ital\nic" + + +def test_strikethrough_across_soft_wrap(): + assert handle_markdown("~~gone\nnow~~") == "gone\nnow" + + +def test_emphasis_does_not_span_blank_lines(): + """A blank line is a paragraph break — emphasis must not match across it.""" + md = "**alpha\n\nbeta**" + assert handle_markdown(md) == md + + +def test_emphasis_does_not_span_whitespace_only_blank_lines(): + md = "**alpha\n \nbeta**" + assert handle_markdown(md) == md + + +# ── Unclosed fenced code blocks (streaming / truncated chunks) ─────────── + + +def test_unclosed_fenced_code_block(): + """An unclosed fence extends to end of input: fence line stripped, content kept.""" + result = handle_markdown("```python\nprint('hi')") + assert result == "print('hi')" + + +def test_unclosed_fenced_code_block_with_prefix(): + result = handle_markdown("Here is code:\n```python\nmodel.train()\n") + assert "`" not in result + assert "python" not in result + assert "model.train()" in result + assert "Here is code:" in result + + +def test_unclosed_fence_protects_content(): + """Markdown inside an unclosed fence must not be stripped.""" + result = handle_markdown("```\n**not stripped**") + assert "**not stripped**" in result + + +def test_bare_unclosed_fence_line(): + """A lone opening fence with no content voices nothing.""" + assert handle_markdown("```python").strip() == "" + + +# ── Parenthesized URLs in links and images ─────────────────────────────── + + +def test_link_with_parenthesized_url(): + md = "[Foo](https://en.wikipedia.org/wiki/Foo_(bar))" + assert handle_markdown(md) == "Foo" + + +def test_image_with_parenthesized_url(): + md = "![alt text](https://example.com/img_(1).png)" + assert handle_markdown(md) == "alt text" + + +def test_link_paren_url_with_surrounding_text(): + md = "See [Foo](https://en.wikipedia.org/wiki/Foo_(bar)) today" + assert handle_markdown(md) == "See Foo today" + + +def test_link_plain_url_still_stops_at_paren(): + """A plain link followed by a parenthetical must not over-match.""" + md = "[here](https://x.com) (see note)" + assert handle_markdown(md) == "here (see note)" + + +# ── Custom phoneme spacing through smart_split ─────────────────────────── + + +@pytest.mark.asyncio +async def test_custom_phoneme_spacing_preserved(): + """Words must not get glued to custom phoneme tokens during normalization. + + smart_split splits on CUSTOM_PHONEMES and rejoins segments with ''.join; + if segment edges get stripped, 'Say [Kokoro](...) please' becomes + 'Say[Kokoro](...)please'. + """ + chunks = [] + async for chunk_text, _, _ in smart_split( + "Say [Kokoro](/kˈOkəɹO/) please.", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "Say [Kokoro](/kˈOkəɹO/) please." in combined + assert "Say[" not in combined + assert ")please" not in combined + + +@pytest.mark.asyncio +async def test_custom_phoneme_spacing_preserved_between_tokens(): + """A whitespace-only segment between two phoneme tokens must survive.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "[Kokoro](/kˈOkəɹO/) [rest](/ɹˈɛst/) now.", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "/) [rest]" in combined + assert "/)[rest]" not in combined + + +# ── Markdown normalization for non-English languages ───────────────────── + + +@pytest.mark.asyncio +async def test_markdown_stripped_for_non_english_language(): + """Markdown syntax is language-independent — it must be stripped even + when the rest of (English-specific) normalization is skipped.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "# Heading\nDas ist **fett** und [Link](https://example.com).", + lang_code="e", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "**" not in combined + assert "#" not in combined + assert "](" not in combined + assert "fett" in combined + assert "Link" in combined + + +@pytest.mark.asyncio +async def test_non_english_markdown_keeps_custom_phonemes(): + """Custom phoneme tokens look like markdown links — they must survive + the markdown pass for non-English languages too.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "Sag [Kokoro](/kˈOkəɹO/) **bitte**.", + lang_code="e", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "[Kokoro](/kˈOkəɹO/)" in combined + assert "**" not in combined + + +@pytest.mark.asyncio +async def test_non_english_markdown_respects_toggle(): + """With markdown_normalization disabled, non-English text is untouched.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "Das ist **fett**.", + lang_code="e", + normalization_options=NormalizationOptions(markdown_normalization=False), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "**fett**" in combined + + +# ── Minor batch: emphasis, headings, links, sentinels, tables ──────────── + + +def test_bold_double_underscore(): + assert handle_markdown("__bold__") == "bold" + + +def test_bold_double_underscore_with_surrounding_text(): + assert handle_markdown("This is __bold__ text") == "This is bold text" + + +def test_closed_atx_heading(): + """ATX headings may close with trailing hashes: '## H ##' -> 'H'.""" + assert handle_markdown("## Heading ##") == "Heading" + + +def test_heading_with_trailing_hash_word_kept(): + """A '#' glued to a word is content, not a closing sequence.""" + assert handle_markdown("# Learning C#") == "Learning C#" + + +def test_heading_after_list_marker(): + """List markers are stripped before headings so '- # H' voices 'H'.""" + assert handle_markdown("- # Heading") == "Heading" + + +def test_heading_after_ordered_list_marker(): + assert handle_markdown("1. ## Heading") == "Heading" + + +def test_empty_link_text_dropped(): + result = handle_markdown("before [](https://example.com) after") + assert "[" not in result + assert "]" not in result + assert "example.com" not in result + assert "before" in result + assert "after" in result + + +def test_sentinel_collision_safe(): + """Literal placeholder-looking input must not corrupt code restoration.""" + md = "weird \x00CB0\x00 input\n```\nsecret code\n```" + result = handle_markdown(md) + assert result.count("secret code") == 1 + + +def test_prose_with_pipes_not_table(): + """2+ pipes in prose without table shape must be preserved.""" + text = "either a | b | c works" + assert handle_markdown(text) == text + + +def test_headerless_table_rows_near_separator(): + """Rows without leading/trailing pipes count as table when adjacent to a + separator row.""" + md = "Name | Age | City\n--- | --- | ---\nAlice | 30 | NYC" + result = handle_markdown(md) + assert "|" not in result + assert "---" not in result + assert "Alice" in result + assert "NYC" in result + + +# --------------------------------------------------------------------------- +# CRLF handling (verifier findings: MULTILINE anchors assume \n) +# --------------------------------------------------------------------------- + + +def test_crlf_closed_atx_heading(): + """Closing-hash headings must strip on CRLF input, not leak '##'.""" + result = handle_markdown("## Heading ##\r\nbody") + assert "#" not in result + assert "Heading" in result + assert "body" in result + + +def test_crlf_blank_line_stops_emphasis(): + """A CRLF blank line is a paragraph break — emphasis must not span it.""" + result = handle_markdown("*start\r\n\r\nend*") + assert "*start" in result + assert "end*" in result + + +def test_crlf_equivalent_to_lf(): + """CRLF input must normalize to the same speech text as LF input.""" + lf = handle_markdown("# Title\n\n**bold** and _em_\n\n- item") + crlf = handle_markdown("# Title\r\n\r\n**bold** and _em_\r\n\r\n- item") + assert crlf == lf + + +# --------------------------------------------------------------------------- +# Pathological-input performance (verifier finding: quadratic emphasis scan) +# --------------------------------------------------------------------------- + + +def test_unclosed_emphasis_flood_is_fast(): + """Repeated unclosed emphasis markers must not scan quadratically. + + 100KB of '*word ' previously took >10s (O(n^2)); bounded emphasis + spans make it linear. Generous ceiling to avoid CI flakiness. + """ + import time + + for marker in ("*word ", "_word ", "__word "): + payload = marker * (100_000 // len(marker)) + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 8.0 + + +# --------------------------------------------------------------------------- +# Blockquoted fences and escaped pipes (verifier minors) +# --------------------------------------------------------------------------- + + +def test_fenced_code_inside_blockquote(): + """A fence opened inside a blockquote must not be garbled by the + inline-code pass pairing backticks across lines.""" + result = handle_markdown("> ```\n> code line\n> ```") + assert "`" not in result + assert "code line" in result + + +def test_escaped_pipes_not_table(): + """Escaped pipes are literal content, not table syntax.""" + result = handle_markdown("use a \\| b \\| c here") + assert result == "use a | b | c here" + + +def test_adversarial_floods_are_fast(): + """Every markdown pass must stay near-linear on pathological input. + + Covers the verifier-found quadratic vectors beyond emphasis: long + dash lines (table separator backtracking), '[' floods (link/image + scan-to-EOS), fence-opener and backtick-pair floods (per-placeholder + str.replace restore loop). + """ + import time + + payloads = [ + "-" * 40_000 + ".", # table separator backtracking + "[word " * (100_000 // 6), # unclosed-bracket link flood + "```\n" * (100_000 // 4), # fence-opener flood -> placeholder restore + "`a`" * (100_000 // 3), # inline-code flood -> placeholder restore + ] + for payload in payloads: + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 8.0, f"slow on {payload[:20]!r}..." + + +def test_emphasis_flood_scales_linearly(): + """Scaling ratio guard: 4x input must cost ~4x time (linear), not + ~16x (quadratic). Ratio bound is generous for CI noise.""" + import time + + def cost(n: int) -> float: + payload = "__word " * (n // 7) + start = time.monotonic() + handle_markdown(payload) + return time.monotonic() - start + + small, large = cost(100_000), cost(400_000) + assert large / max(small, 1e-3) < 9.0 + + +def test_newline_and_whitespace_floods_are_fast(): + """List-marker patterns must not scan quadratically on blank-line + floods ('^(\\s*)' + MULTILINE consumed all following newlines from + every line anchor; 100KB of newlines took 65s).""" + import time + + for payload in ["\n" * 100_000, "\r\n" * 50_000, " \n" * 50_000]: + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 8.0 + + +def test_long_wordrun_through_normalize_text_is_fast(): + """URL_PATTERN must not scan quadratically on long word-char runs + with no dot-TLD (100KB single line previously hung normalize_text).""" + import time + + from api.src.services.text_processing.normalizer import ( + normalize_text, + ) + from api.src.structures.schemas import NormalizationOptions + + for payload in ["a" * 100_000, "a." * 50_000, "a@" * 50_000]: + start = time.monotonic() + normalize_text(payload, NormalizationOptions()) + assert time.monotonic() - start < 8.0 + + +# --------------------------------------------------------------------------- +# Borderless tables (Gemini finding: rows beyond the first, and 2-col tables, +# were left with raw pipes by the old anchored/adjacent heuristic) +# --------------------------------------------------------------------------- + + +def test_borderless_multirow_table_all_rows_depiped(): + md = "Name | Age | City\n--- | --- | ---\nAlice | 30 | NYC\nBob | 25 | LA" + result = handle_markdown(md) + assert "|" not in result + assert "---" not in result + for token in ("Alice", "Bob", "NYC", "LA"): + assert token in result + + +def test_borderless_two_column_table_depiped(): + """2-column borderless rows have only one pipe — the old count>=2 gate + skipped them entirely.""" + md = "Key | Value\n--- | ---\nfoo | bar" + result = handle_markdown(md) + assert "|" not in result + assert "foo" in result and "bar" in result + + +def test_lone_prose_pipes_still_intact(): + """A pipe-bearing prose line with no adjacent separator row is not a table.""" + text = "either a | b | c works" + assert handle_markdown(text) == text diff --git a/docker-bake.hcl b/docker-bake.hcl index 70e49ba8..4b6583f1 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -133,11 +133,17 @@ target "cpu-arm64" { ] } +# CUDA base images pinned by digest (index/manifest-list digests) to prevent +# supply-chain substitution of the mutable version tags. The Dockerfile defaults +# stay unpinned so plain `docker build` / compose builds keep working. +# Refresh with: docker buildx imagetools inspect nvcr.io/nvidia/cuda: target "gpu-amd64" { inherits = ["_gpu_base"] platforms = ["linux/amd64"] args = { CUDA_VERSION = "12.6.3" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04@sha256:50efab398f76258daa91ceebb33b6467e40217c67ea44fb5a2cebc6be7d9cce3" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04@sha256:8aef630a54bc5c5146ae5ce68e6af5caa3df0fb690bb91544175c91f307e4356" } # Per-arch tag carries the wheel variant so it parallels gpu-cu128-amd64. # The published manifest still resolves to :VERSION / :VERSION-cu126 via release.yml. @@ -151,6 +157,8 @@ target "gpu-arm64" { platforms = ["linux/arm64"] args = { CUDA_VERSION = "12.9.1" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.9.1-cudnn-devel-ubuntu24.04@sha256:a2e1e2360c85298ac47ec2543b406ab1e8cec42e31ee47e4d32140ebc82e1067" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.9.1-cudnn-runtime-ubuntu24.04@sha256:d02c4310b6d57ca0b16cd80298bdb33a74187baafe2eccd8a6a16180ddc90802" } # aarch64 uses cu129 wheels (no cu126 aarch64 wheels exist on pytorch.org). tags = [ @@ -167,6 +175,8 @@ target "gpu-cu128-amd64" { # 12.8.x is the first CUDA toolkit with Blackwell (sm_120) support and is # what the cu128 torch wheels are built against. Keep base + wheel aligned. CUDA_VERSION = "12.8.1" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04@sha256:24c8e3581ea6330038b0d374920721983312627f8adbfcf390bdb4b399d280ed" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04@sha256:ac55d124da4882b497f732d8dfd9a702d5447a5f29d08d56da6f64f0a1eb34bc" GPU_EXTRA = "gpu-cu128" } tags = [ diff --git a/docker/gpu/Dockerfile.optimized b/docker/gpu/Dockerfile.optimized index cd45c7ed..0d1804d8 100644 --- a/docker/gpu/Dockerfile.optimized +++ b/docker/gpu/Dockerfile.optimized @@ -2,8 +2,14 @@ # The arm64 bump is required because pytorch.org/whl/cu126 has no aarch64 wheels. ARG CUDA_VERSION=12.6.3 +# Full base-image refs. Defaults keep plain `docker build` / compose builds working +# off the mutable version tags; docker-bake.hcl overrides these with tag@digest +# pins so release builds are immune to base-image substitution. +ARG CUDA_BUILDER_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 +ARG CUDA_RUNTIME_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu24.04 + # Stage 1: Builder - Use devel image for compilation -FROM --platform=$BUILDPLATFORM nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS builder +FROM --platform=$BUILDPLATFORM ${CUDA_BUILDER_IMAGE} AS builder # Install Python and build dependencies RUN apt-get update -y && \ @@ -33,7 +39,7 @@ RUN uv venv --python 3.10 && \ uv sync --extra ${GPU_EXTRA} --no-cache --no-install-project # Stage 2: Runtime - Use smaller runtime image -FROM --platform=$BUILDPLATFORM nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu24.04 +FROM --platform=$BUILDPLATFORM ${CUDA_RUNTIME_IMAGE} # Install runtime dependencies + uv (needed by entrypoint.sh) RUN apt-get update -y && \