From f7045a1aa318a485fcacfe3df047dfd190494486 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 21 Feb 2026 16:40:53 -0600 Subject: [PATCH 01/17] feat: add markdown-aware TTS text normalization (#1) Add handle_markdown() pre-processing pass that strips markdown formatting (headings, bold, italic, links, code blocks, tables, etc.) before TTS synthesis. Preserves code block contents literally and avoids false positives on snake_case identifiers. - New: handle_markdown() function in normalizer.py - New: markdown_normalization toggle in NormalizationOptions (default True) - New: test_normalizer.py with 24 test cases including mixed LLM output Rebase note (v0.6.0): the original commit also stripped trailing whitespace from normalize_text() output; that was dropped because upstream's smart_split() now rejoins custom-phoneme segments with "".join() and relies on boundary whitespace being preserved (test_smart_custom_phenomes / test_smart_split_only_phenomes). Co-authored-by: Claude Opus 4.6 Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 107 ++++++++- .../text_processing/test_normalizer.py | 224 ++++++++++++++++++ api/src/structures/schemas.py | 12 +- 3 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 api/src/services/text_processing/test_normalizer.py diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 13d76763..8f01830f 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -1,7 +1,10 @@ """ -Text normalization module for TTS processing. +Text normalization module for TTS processing — forked from remsky/Kokoro-FastAPI. Handles various text formats including URLs, emails, numbers, money, and special characters. Converts them into a format suitable for text-to-speech processing. + +Upstream: https://github.com/remsky/Kokoro-FastAPI/blob/master/api/src/services/text_processing/normalizer.py +Delta: handle_markdown() function + markdown_normalization toggle in normalize_text() """ import math @@ -188,6 +191,101 @@ re.IGNORECASE, ) +# --------------------------------------------------------------------------- +# Markdown normalization patterns (added by navi-os fork) +# --------------------------------------------------------------------------- +# Order matters: fenced code blocks first, then inline, then structural. +# Bold before italic to avoid partial matches on **. +_MD_FENCED_CODE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) +_MD_INLINE_CODE = re.compile(r"`([^`]+)`") +_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]+\)") +_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)") +_MD_BOLD = re.compile(r"\*\*(.+?)\*\*") +# Italic with * — only match when not preceded/followed by word chars to avoid +# false positives inside URLs or filenames. +_MD_ITALIC_STAR = re.compile(r"(?\s?", re.MULTILINE) +_MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) +_MD_UNORDERED_LIST = re.compile(r"^(\s*)[-*+]\s+", re.MULTILINE) +_MD_ORDERED_LIST = re.compile(r"^(\s*)\d+\.\s+", re.MULTILINE) +# Table separator rows like |---|---| +_MD_TABLE_SEP_ROW = re.compile(r"^\s*\|?[\s:]*-{3,}[\s:|-]*\|?\s*$", re.MULTILINE) + + +def handle_markdown(text: str) -> str: + """Strip markdown formatting so TTS reads content, not syntax. + + Designed as a pre-processing pass: runs BEFORE email/URL normalization + so that markdown link syntax ``[text](url)`` is reduced to ``text`` + before the URL handler sees bare URLs. + """ + # --- Phase 1: Protect code content from markdown stripping --- + # Extract fenced code blocks and inline code into placeholders so that + # markdown syntax inside code (e.g. **bold**) is preserved literally. + code_blocks: list[str] = [] + + def _save_fenced(m: re.Match) -> str: + code_blocks.append(m.group(1)) + return f"\x00CB{len(code_blocks) - 1}\x00" + + text = _MD_FENCED_CODE.sub(_save_fenced, text) + + inline_codes: list[str] = [] + + def _save_inline(m: re.Match) -> str: + inline_codes.append(m.group(1)) + return f"\x00IC{len(inline_codes) - 1}\x00" + + text = _MD_INLINE_CODE.sub(_save_inline, text) + + # --- Phase 2: Strip markdown formatting --- + # Images — keep alt text + text = _MD_IMAGE.sub(r"\1", text) + # Links — keep link text + text = _MD_LINK.sub(r"\1", text) + # Bold (** before * to avoid partial match) + text = _MD_BOLD.sub(r"\1", text) + # Italic (* and _) + text = _MD_ITALIC_STAR.sub(r"\1", text) + text = _MD_ITALIC_UNDER.sub(r"\1", text) + # Strikethrough + text = _MD_STRIKETHROUGH.sub(r"\1", text) + # Headings — strip leading hashes + text = _MD_HEADING.sub("", text) + # Blockquotes — strip leading > + text = _MD_BLOCKQUOTE.sub("", text) + # Horizontal rules — remove entire line + text = _MD_HORIZONTAL_RULE.sub("", text) + # List markers — strip marker, keep text + text = _MD_UNORDERED_LIST.sub(r"\1", text) + text = _MD_ORDERED_LIST.sub(r"\1", text) + # Table separator rows — remove entirely + text = _MD_TABLE_SEP_ROW.sub("", text) + # Table pipes — only replace on lines that look like table rows (2+ pipes). + # This preserves non-table pipes like "true | false". + lines = text.split("\n") + for i, line in enumerate(lines): + if line.count("|") >= 2: + lines[i] = line.replace("|", " ") + text = "\n".join(lines) + + # --- Phase 3: Restore protected code content --- + for idx, content in enumerate(code_blocks): + text = text.replace(f"\x00CB{idx}\x00", content) + for idx, content in enumerate(inline_codes): + text = text.replace(f"\x00IC{idx}\x00", content) + + return text + + +# --------------------------------------------------------------------------- +# Upstream handlers (unchanged from v0.2.4) +# --------------------------------------------------------------------------- + INFLECT_ENGINE = inflect.engine() @@ -410,6 +508,13 @@ def handle_time(t: re.Match[str]) -> str: def normalize_text(text: str, normalization_options: NormalizationOptions) -> str: """Normalize text for TTS processing""" + # Handle markdown formatting FIRST — strips syntax like **bold**, [link](url), + # ```code```, etc. before other handlers see the raw symbols. + # Must run before email/URL normalization: markdown links contain URLs that + # should be reduced to link text, not spelled out. + if normalization_options.markdown_normalization: + text = handle_markdown(text) + # Handle email addresses first if enabled if normalization_options.email_normalization: text = EMAIL_PATTERN.sub(handle_email, text) diff --git a/api/src/services/text_processing/test_normalizer.py b/api/src/services/text_processing/test_normalizer.py new file mode 100644 index 00000000..2478b2ee --- /dev/null +++ b/api/src/services/text_processing/test_normalizer.py @@ -0,0 +1,224 @@ +"""Unit tests for handle_markdown() — the navi-os fork of Kokoro's normalizer. + +Tests run standalone: pytest deploy/desktop/kokoro/test_normalizer.py -v +No Docker or Kokoro dependencies required — we import handle_markdown directly. +""" + +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock + +# --------------------------------------------------------------------------- +# Shim: fake Kokoro's third-party and internal dependencies so we can import +# normalizer.py without the full Kokoro environment. We only need +# handle_markdown(), which uses nothing beyond the stdlib `re` module. +# --------------------------------------------------------------------------- + +# Third-party libs used at module level in normalizer.py +for _dep in ("inflect", "numpy", "numpy.number", "torch"): + if _dep not in sys.modules: + sys.modules[_dep] = MagicMock() + +# Internal Kokoro package hierarchy for the relative import: +# from ...structures.schemas import NormalizationOptions +_fake_schemas = types.ModuleType("api.src.structures.schemas") + + +class _StubNormalizationOptions: + pass + + +_fake_schemas.NormalizationOptions = _StubNormalizationOptions # type: ignore[attr-defined] + +for mod_name in [ + "api", + "api.src", + "api.src.services", + "api.src.services.text_processing", + "api.src.structures", +]: + if mod_name not in sys.modules: + sys.modules[mod_name] = types.ModuleType(mod_name) + +sys.modules["api.src.structures.schemas"] = _fake_schemas + +# Wire up sub-module attributes so dotted access works +sys.modules["api"].src = sys.modules["api.src"] # type: ignore[attr-defined] +sys.modules["api.src"].services = sys.modules["api.src.services"] # type: ignore[attr-defined] +sys.modules["api.src"].structures = sys.modules["api.src.structures"] # type: ignore[attr-defined] +sys.modules["api.src.services"].text_processing = sys.modules["api.src.services.text_processing"] # type: ignore[attr-defined] +sys.modules["api.src.structures"].schemas = _fake_schemas # type: ignore[attr-defined] + +# Import normalizer with correct __package__ for relative imports +import importlib.util + +_spec = importlib.util.spec_from_file_location( + "api.src.services.text_processing.normalizer", + Path(__file__).resolve().parent / "normalizer.py", + submodule_search_locations=[], +) +_mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] +_mod.__package__ = "api.src.services.text_processing" # type: ignore[attr-defined] +sys.modules[_spec.name] = _mod # type: ignore[union-attr] +_spec.loader.exec_module(_mod) # type: ignore[union-attr] + +handle_markdown = _mod.handle_markdown # type: ignore[union-attr] + +# ── Tests ──────────────────────────────────────────────────────────────── + + +def test_bold(): + assert handle_markdown("**bold text**") == "bold text" + + +def test_italic_star(): + assert handle_markdown("*italic*") == "italic" + + +def test_italic_underscore(): + assert handle_markdown("_italic_") == "italic" + + +def test_snake_case_preserved(): + assert handle_markdown("snake_case_var") == "snake_case_var" + + +def test_strikethrough(): + assert handle_markdown("~~deleted~~") == "deleted" + + +def test_inline_code(): + assert handle_markdown("`code`") == "code" + + +def test_fenced_code_block(): + result = handle_markdown("```python\nprint(\"hi\")\n```") + assert result.strip() == 'print("hi")' + + +def test_link(): + assert handle_markdown("[click here](https://example.com)") == "click here" + + +def test_image(): + assert handle_markdown("![alt text](image.png)") == "alt text" + + +def test_heading(): + assert handle_markdown("### My Heading") == "My Heading" + + +def test_blockquote(): + assert handle_markdown("> quoted text") == "quoted text" + + +def test_horizontal_rule(): + assert handle_markdown("---").strip() == "" + + +def test_unordered_list(): + assert handle_markdown("- item one") == "item one" + + +def test_ordered_list(): + assert handle_markdown("1. first item") == "first item" + + +def test_table(): + md = "| Name | Age |\n| --- | --- |\n| Alice | 30 |" + result = handle_markdown(md) + # Separator row removed; pipes replaced with spaces on table lines + assert "Name" in result + assert "Age" in result + assert "Alice" in result + assert "30" in result + # No pipe characters should remain + assert "|" not in result + # Separator row content should be gone + assert "---" not in result + + +def test_non_table_pipe_preserved(): + assert handle_markdown("true | false") == "true | false" + + +def test_nested_bold_italic(): + assert handle_markdown("***bold italic***") == "bold italic" + + +def test_code_block_preserves_markdown(): + """Markdown inside fenced code blocks must NOT be stripped.""" + result = handle_markdown("```\n**not stripped**\n```") + assert "**not stripped**" in result + + +def test_inline_code_preserves_markdown(): + """Markdown inside inline code must NOT be stripped.""" + result = handle_markdown("`**bold**`") + assert result.strip() == "**bold**" + + +def test_mixed_llm_output(): + """Integration test: a realistic LLM response with mixed markdown.""" + md = ( + "### Summary\n\n" + "This is **important** information about *machine learning*.\n\n" + "Key points:\n" + "- First, read the [documentation](https://docs.example.com)\n" + "- Then, run `pip install torch`\n\n" + "Here is an example:\n\n" + "```python\nmodel.train()\n```\n\n" + "> Note: this is experimental\n\n" + "| Metric | Value |\n" + "| --- | --- |\n" + "| Accuracy | 95% |\n" + ) + result = handle_markdown(md) + # Formatting artifacts should be gone + assert "**" not in result + assert "###" not in result + assert "](http" not in result + assert "| --- |" not in result + # Content should remain + assert "important" in result + assert "machine learning" in result + assert "documentation" in result + assert "pip install torch" in result + assert "model.train()" in result + assert "experimental" in result + assert "Accuracy" in result + assert "95%" in result + + +def test_multiple_code_blocks(): + """Multiple fenced code blocks should all be protected.""" + md = "```\n**block1**\n```\nSome **bold** text\n```\n*block2*\n```" + result = handle_markdown(md) + assert "**block1**" in result + assert "*block2*" in result + assert "bold" in result + assert "**bold**" not in result + + +def test_heading_levels(): + """All heading levels (1-6) should be stripped.""" + for i in range(1, 7): + hashes = "#" * i + assert handle_markdown(f"{hashes} Heading {i}") == f"Heading {i}" + + +def test_unordered_list_markers(): + """All unordered list markers (-, *, +) should be stripped.""" + assert handle_markdown("- dash item") == "dash item" + # * is also an italic marker, but at start of line with space it's a list + assert handle_markdown("+ plus item") == "plus item" + + +def test_bold_with_surrounding_text(): + assert handle_markdown("This is **bold** in context") == "This is bold in context" + + +def test_link_with_surrounding_text(): + result = handle_markdown("Click [here](https://x.com) now") + assert result == "Click here now" diff --git a/api/src/structures/schemas.py b/api/src/structures/schemas.py index 4fd6ca0e..f7037dc0 100644 --- a/api/src/structures/schemas.py +++ b/api/src/structures/schemas.py @@ -1,4 +1,9 @@ -from email.policy import default +"""Kokoro-FastAPI schemas — forked from upstream to add markdown_normalization toggle. + +Upstream: https://github.com/remsky/Kokoro-FastAPI/blob/master/api/src/structures/schemas.py +Delta: NormalizationOptions.markdown_normalization field (default True) +""" + from enum import Enum from typing import List, Literal, Optional, Union @@ -48,6 +53,11 @@ class NormalizationOptions(BaseModel): default=True, description="Normalizes input text to make it easier for the model to say", ) + markdown_normalization: bool = Field( + default=True, + description="Strips markdown formatting for cleaner TTS output " + "(e.g., **bold** -> bold, [link](url) -> link)", + ) unit_normalization: bool = Field( default=False, description="Transforms units like 10KB to 10 kilobytes" ) From d8825776ab77dc7af05c6a9acc25cb4b9264b360 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 16 Jul 2026 12:30:14 -0400 Subject: [PATCH 02/17] security: pin CUDA base images by digest in bake config Re-implementation of the fork's base-image digest pin against upstream's multi-stage docker/gpu/Dockerfile.optimized + docker-bake.hcl layout (the old single-stage docker/gpu/Dockerfile was removed upstream). - Dockerfile.optimized gains CUDA_BUILDER_IMAGE / CUDA_RUNTIME_IMAGE args defaulting to the unpinned ${CUDA_VERSION} tags, so plain docker/compose builds keep working. - docker-bake.hcl overrides them with tag@sha256 index digests for gpu-amd64 (12.6.3), gpu-arm64 (12.9.1), and gpu-cu128-amd64 (12.8.1), preventing supply-chain substitution of the mutable version tags. Digests resolved 2026-07-16 via docker buildx imagetools inspect. Co-Authored-By: Claude Fable 5 --- docker-bake.hcl | 10 ++++++++++ docker/gpu/Dockerfile.optimized | 10 ++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docker-bake.hcl b/docker-bake.hcl index 70e49ba8..4b6583f1 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -133,11 +133,17 @@ target "cpu-arm64" { ] } +# CUDA base images pinned by digest (index/manifest-list digests) to prevent +# supply-chain substitution of the mutable version tags. The Dockerfile defaults +# stay unpinned so plain `docker build` / compose builds keep working. +# Refresh with: docker buildx imagetools inspect nvcr.io/nvidia/cuda: target "gpu-amd64" { inherits = ["_gpu_base"] platforms = ["linux/amd64"] args = { CUDA_VERSION = "12.6.3" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04@sha256:50efab398f76258daa91ceebb33b6467e40217c67ea44fb5a2cebc6be7d9cce3" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04@sha256:8aef630a54bc5c5146ae5ce68e6af5caa3df0fb690bb91544175c91f307e4356" } # Per-arch tag carries the wheel variant so it parallels gpu-cu128-amd64. # The published manifest still resolves to :VERSION / :VERSION-cu126 via release.yml. @@ -151,6 +157,8 @@ target "gpu-arm64" { platforms = ["linux/arm64"] args = { CUDA_VERSION = "12.9.1" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.9.1-cudnn-devel-ubuntu24.04@sha256:a2e1e2360c85298ac47ec2543b406ab1e8cec42e31ee47e4d32140ebc82e1067" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.9.1-cudnn-runtime-ubuntu24.04@sha256:d02c4310b6d57ca0b16cd80298bdb33a74187baafe2eccd8a6a16180ddc90802" } # aarch64 uses cu129 wheels (no cu126 aarch64 wheels exist on pytorch.org). tags = [ @@ -167,6 +175,8 @@ target "gpu-cu128-amd64" { # 12.8.x is the first CUDA toolkit with Blackwell (sm_120) support and is # what the cu128 torch wheels are built against. Keep base + wheel aligned. CUDA_VERSION = "12.8.1" + CUDA_BUILDER_IMAGE = "nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04@sha256:24c8e3581ea6330038b0d374920721983312627f8adbfcf390bdb4b399d280ed" + CUDA_RUNTIME_IMAGE = "nvcr.io/nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04@sha256:ac55d124da4882b497f732d8dfd9a702d5447a5f29d08d56da6f64f0a1eb34bc" GPU_EXTRA = "gpu-cu128" } tags = [ diff --git a/docker/gpu/Dockerfile.optimized b/docker/gpu/Dockerfile.optimized index cd45c7ed..0d1804d8 100644 --- a/docker/gpu/Dockerfile.optimized +++ b/docker/gpu/Dockerfile.optimized @@ -2,8 +2,14 @@ # The arm64 bump is required because pytorch.org/whl/cu126 has no aarch64 wheels. ARG CUDA_VERSION=12.6.3 +# Full base-image refs. Defaults keep plain `docker build` / compose builds working +# off the mutable version tags; docker-bake.hcl overrides these with tag@digest +# pins so release builds are immune to base-image substitution. +ARG CUDA_BUILDER_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 +ARG CUDA_RUNTIME_IMAGE=nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu24.04 + # Stage 1: Builder - Use devel image for compilation -FROM --platform=$BUILDPLATFORM nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS builder +FROM --platform=$BUILDPLATFORM ${CUDA_BUILDER_IMAGE} AS builder # Install Python and build dependencies RUN apt-get update -y && \ @@ -33,7 +39,7 @@ RUN uv venv --python 3.10 && \ uv sync --extra ${GPU_EXTRA} --no-cache --no-install-project # Stage 2: Runtime - Use smaller runtime image -FROM --platform=$BUILDPLATFORM nvcr.io/nvidia/cuda:${CUDA_VERSION}-cudnn-runtime-ubuntu24.04 +FROM --platform=$BUILDPLATFORM ${CUDA_RUNTIME_IMAGE} # Install runtime dependencies + uv (needed by entrypoint.sh) RUN apt-get update -y && \ From cb514d2ac4bf6b0a5798acb03bf2f5fbc0c903bf Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 21 Feb 2026 16:42:27 -0600 Subject: [PATCH 03/17] security: hardened GHCR release workflow with supply-chain protections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace upstream's multi-arch bake release workflow with a GPU-only, amd64-only build that includes: - Pinned action SHAs (all 11 actions) - step-security/harden-runner (audit mode) - Sigstore build provenance attestation - Trivy CRITICAL+HIGH container scan → GitHub Security tab - CycloneDX SBOM attached to GitHub Release Triggers on tag push (v*) or manual workflow_dispatch. Publishes to ghcr.io/fieldnote-echo/kokoro-fastapi-gpu. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/release.yml | 372 ++++++++++------------------------ 1 file changed, 112 insertions(+), 260 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f21d8b5..d619842a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,295 +1,147 @@ -name: Create Release and Publish Docker Images +# Fieldnote-Echo/Kokoro-FastAPI — Hardened GHCR Release +# Replaces upstream's multi-arch bake workflow with a GPU-only, amd64-only +# build that includes supply-chain hardening (pinned actions, Sigstore +# attestation, Trivy scan, CycloneDX SBOM). +# +# Incorporates from upstream's reworked release workflow: concurrency group, +# duplicate-release guard, ubuntu-24.04 runner, and the new multi-stage +# docker/gpu/Dockerfile.optimized build context. +# +# Triggers: tag push (v*) or manual dispatch. + +name: Release GPU Image on: push: - branches: - - release # Auto-trigger only on release branch - paths-ignore: - - '**.md' - - 'docs/**' - workflow_dispatch: # Manual trigger - explicitly specify branch - inputs: - branch_name: - description: 'Branch to build from (required)' - required: true - type: string + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: write # GitHub Release creation + packages: write # GHCR push + id-token: write # Sigstore OIDC + security-events: write # SARIF upload + attestations: write # Build provenance concurrency: - group: release-${{ github.event_name == 'workflow_dispatch' && inputs.branch_name || github.ref_name }} + group: release-${{ github.ref_name }} cancel-in-progress: false jobs: - prepare-release: + build-and-release: runs-on: ubuntu-24.04 - outputs: - version: ${{ steps.get-version.outputs.version }} - version_tag: ${{ steps.get-version.outputs.version_tag }} - source_ref: ${{ steps.resolve-ref.outputs.source_ref }} + env: + IMAGE: ghcr.io/fieldnote-echo/kokoro-fastapi-gpu steps: - - name: Resolve source ref - id: resolve-ref - run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "source_ref=${{ inputs.branch_name }}" >> $GITHUB_OUTPUT - else - echo "source_ref=${{ github.ref_name }}" >> $GITHUB_OUTPUT - fi - - - name: Checkout repository - uses: actions/checkout@v5 + # ── Runner hardening ────────────────────────────────────────────── + - name: Harden runner + uses: step-security/harden-runner@5ef0c079ce82195b2a36a210272d6b661572d83e # v2.14.2 with: - ref: ${{ steps.resolve-ref.outputs.source_ref }} - fetch-depth: 0 + egress-policy: audit - - name: Get version from VERSION file - id: get-version - run: | - VERSION_PLAIN=$(cat VERSION) - BRANCH_NAME="${{ steps.resolve-ref.outputs.source_ref }}" - - if [[ "$BRANCH_NAME" == "release" ]]; then - echo "version=${VERSION_PLAIN}" >> $GITHUB_OUTPUT - echo "version_tag=v${VERSION_PLAIN}" >> $GITHUB_OUTPUT - else - SAFE_BRANCH=$(echo "$BRANCH_NAME" | tr '/' '-' | tr '[:upper:]' '[:lower:]') - echo "version=${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT - echo "version_tag=v${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT - fi + # ── Checkout ────────────────────────────────────────────────────── + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Check tag does not exist + # ── Guard: don't re-publish an existing release ─────────────────── + - name: Check release does not exist + if: startsWith(github.ref, 'refs/tags/') + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - TAG_NAME="${{ steps.get-version.outputs.version_tag }}" - git fetch --tags - if git rev-parse "$TAG_NAME" >/dev/null 2>&1; then - echo "::error::Tag $TAG_NAME already exists. Please increment the version in the VERSION file." + if gh release view "${{ github.ref_name }}" --repo "${{ github.repository }}" >/dev/null 2>&1; then + echo "::error::Release ${{ github.ref_name }} already exists. Push a new tag to publish a new release." exit 1 fi - build-images: - needs: prepare-release - permissions: - packages: write - env: - DOCKER_BUILDKIT: 1 - BUILDKIT_STEP_LOG_MAX_SIZE: 10485760 - VERSION: ${{ needs.prepare-release.outputs.version_tag }} - REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} - OWNER: ${{ vars.OWNER || 'remsky' }} - REPO: ${{ vars.REPO || 'kokoro-fastapi' }} - strategy: - matrix: - include: - - build_target: "cpu" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "gpu" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "gpu-cu128" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - - build_target: "cpu" - platform: "linux/arm64" - runs_on: "ubuntu-24.04-arm" - - build_target: "gpu" - platform: "linux/arm64" - runs_on: "ubuntu-24.04-arm" - - build_target: "rocm" - platform: "linux/amd64" - runs_on: "ubuntu-24.04" - runs-on: ${{ matrix.runs_on }} - steps: - - name: Checkout repository - uses: actions/checkout@v5 - with: - ref: ${{ needs.prepare-release.outputs.source_ref }} - + # ── Free disk space (CUDA images are ~12 GB) ────────────────────── - name: Free disk space run: | - echo "Listing current disk space" - df -h - echo "Cleaning up disk space..." - sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc docker system prune -af - echo "Disk space after cleanup" - df -h - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - with: - driver-opts: | - image=moby/buildkit:v0.21.1 - network=host + # ── Docker Buildx ───────────────────────────────────────────────── + - name: Set up Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 + # ── GHCR login ──────────────────────────────────────────────────── + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push single-platform image - run: | - PLATFORM="${{ matrix.platform }}" - BUILD_TARGET="${{ matrix.build_target }}" - VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" - - echo "Building ${PLATFORM} image for ${BUILD_TARGET} version ${VERSION_TAG}" - - TARGET="${BUILD_TARGET}-$(echo ${PLATFORM} | cut -d'/' -f2)" - echo "Using bake target: $TARGET" - - # Bake reads REVISION/CREATED as HCL variables; both populate - # org.opencontainers.image.* labels and annotations on the per-arch push. - export REVISION="${GITHUB_SHA}" - export CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - docker buildx bake $TARGET --push --progress=plain \ - --set "*.cache-from=type=gha,scope=${TARGET}" \ - --set "*.cache-to=type=gha,mode=max,scope=${TARGET}" - - create-manifests: - needs: [prepare-release, build-images] - runs-on: ubuntu-24.04 - permissions: - packages: write - env: - REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} - OWNER: ${{ vars.OWNER || 'remsky' }} - REPO: ${{ vars.REPO || 'kokoro-fastapi' }} - strategy: - matrix: - build_target: ["cpu", "gpu", "gpu-cu128", "rocm"] - steps: - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 + # ── Image metadata (semver tags + latest) ───────────────────────── + - name: Extract metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create multi-platform manifest - run: | - VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" - TARGET="${{ matrix.build_target }}" - REGISTRY="${{ env.REGISTRY }}" - OWNER="${{ env.OWNER }}" - REPO="${{ env.REPO }}" - - # Resolve the package name, manifest tag(s), and source images per target. - # - # Tag layout per target: - # - cpu: :VERSION + :latest (multi-arch) - # - gpu: :VERSION + :VERSION-cu126 + :latest + :latest-cu126 - # (the -cu126 aliases make the wheel variant explicit so - # Maxwell/Pascal users can pin it across future default flips) - # - gpu-cu128: :VERSION-cu128 + :latest-cu128 (amd64 only) - # - rocm: :VERSION + :latest (amd64 only) - # - # MANIFEST_TAGS / LATEST_TAGS are space-separated lists; all entries point at - # the same SOURCES so the underlying blobs are deduplicated by digest. - # - # TITLE / DESCRIPTION are attached as index-level OCI annotations. - # GHCR reads org.opencontainers.image.description from the index manifest - # for multi-arch packages, so the per-arch labels set by bake are not - # enough on their own. - case "$TARGET" in - gpu-cu128) - PACKAGE="${REPO}-gpu" - MANIFEST_TAGS="${VERSION_TAG}-cu128" - LATEST_TAGS="latest-cu128" - SOURCES=("${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu128-amd64") - TITLE="Kokoro-FastAPI (GPU, cu128 / Blackwell)" - DESCRIPTION="Kokoro TTS served via FastAPI. NVIDIA GPU build with CUDA 12.8 + cu128 torch wheels for RTX 50-series (Blackwell, sm_120). amd64 only." - ;; - gpu) - PACKAGE="${REPO}-gpu" - MANIFEST_TAGS="${VERSION_TAG} ${VERSION_TAG}-cu126" - LATEST_TAGS="latest latest-cu126" - # Per-arch sources carry their wheel variant (cu126 amd64, cu129 arm64); - # both feed the same multi-arch manifest tags above. - SOURCES=( - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu126-amd64" - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-cu129-arm64" - ) - TITLE="Kokoro-FastAPI (GPU)" - DESCRIPTION="Kokoro TTS served via FastAPI. NVIDIA GPU build, multi-arch (CUDA 12.6 + cu126 wheels on amd64, CUDA 12.9 + cu129 wheels on arm64). See the -cu128 tag for Blackwell." - ;; - rocm) - PACKAGE="${REPO}-rocm" - MANIFEST_TAGS="${VERSION_TAG}" - LATEST_TAGS="latest" - SOURCES=("${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-amd64") - TITLE="Kokoro-FastAPI (ROCm)" - DESCRIPTION="Kokoro TTS served via FastAPI. AMD ROCm build. amd64 only." - ;; - *) - PACKAGE="${REPO}-${TARGET}" - MANIFEST_TAGS="${VERSION_TAG}" - LATEST_TAGS="latest" - SOURCES=( - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-amd64" - "${REGISTRY}/${OWNER}/${PACKAGE}:${VERSION_TAG}-arm64" - ) - TITLE="Kokoro-FastAPI (CPU)" - DESCRIPTION="Kokoro TTS served via FastAPI. CPU build, multi-arch (amd64 + arm64)." - ;; - esac - - # Common index annotations. `index:` is required so these land on the - # index manifest itself, not on per-platform descriptors inside it. - SOURCE_URL="https://github.com/${OWNER}/Kokoro-FastAPI" - CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - REVISION="${GITHUB_SHA}" - ANNOTATIONS=( - --annotation "index:org.opencontainers.image.title=${TITLE}" - --annotation "index:org.opencontainers.image.description=${DESCRIPTION}" - --annotation "index:org.opencontainers.image.source=${SOURCE_URL}" - --annotation "index:org.opencontainers.image.url=${SOURCE_URL}" - --annotation "index:org.opencontainers.image.licenses=Apache-2.0" - --annotation "index:org.opencontainers.image.version=${VERSION_TAG}" - --annotation "index:org.opencontainers.image.revision=${REVISION}" - --annotation "index:org.opencontainers.image.created=${CREATED}" - ) + images: ${{ env.IMAGE }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix= + + # ── Build & push ────────────────────────────────────────────────── + # CUDA base images are pinned by digest, mirroring docker-bake.hcl's + # gpu-amd64 target (CUDA 12.6.3). Keep these in sync with the bake file; + # refresh with: docker buildx imagetools inspect nvcr.io/nvidia/cuda: + - name: Build and push + id: build + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + file: docker/gpu/Dockerfile.optimized + platforms: linux/amd64 + push: true + build-args: | + CUDA_VERSION=12.6.3 + CUDA_BUILDER_IMAGE=nvcr.io/nvidia/cuda:12.6.3-cudnn-devel-ubuntu24.04@sha256:50efab398f76258daa91ceebb33b6467e40217c67ea44fb5a2cebc6be7d9cce3 + CUDA_RUNTIME_IMAGE=nvcr.io/nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04@sha256:8aef630a54bc5c5146ae5ce68e6af5caa3df0fb690bb91544175c91f307e4356 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ── Sigstore build provenance ───────────────────────────────────── + - name: Attest build provenance + uses: actions/attest-build-provenance@96278af6caaf10aea03fd8d33a09a777ca52d62f # v3.2.0 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true - for TAG in $MANIFEST_TAGS; do - docker buildx imagetools create \ - "${ANNOTATIONS[@]}" \ - -t ${REGISTRY}/${OWNER}/${PACKAGE}:${TAG} \ - "${SOURCES[@]}" - done + # ── Trivy container scan ────────────────────────────────────────── + - name: Trivy vulnerability scan + uses: aquasecurity/trivy-action@c1824fd6edce30d7ab345a9989de00bbd46ef284 # 0.34.0 + with: + image-ref: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }} + format: sarif + output: trivy-results.sarif + severity: CRITICAL,HIGH - # Publish the rolling tag(s) only for clean release tags (no branch suffix). - if [[ "$VERSION_TAG" != *"-"* ]]; then - for TAG in $LATEST_TAGS; do - docker buildx imagetools create \ - "${ANNOTATIONS[@]}" \ - -t ${REGISTRY}/${OWNER}/${PACKAGE}:${TAG} \ - "${SOURCES[@]}" - done - fi + - name: Upload Trivy SARIF + uses: github/codeql-action/upload-sarif@9e907b5e64f6b83e7804b09294d44122997950d6 # v4.32.3 + with: + sarif_file: trivy-results.sarif - create-release: - needs: [prepare-release, create-manifests] - runs-on: ubuntu-24.04 - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v5 + # ── CycloneDX SBOM ─────────────────────────────────────────────── + - name: Generate SBOM + uses: anchore/sbom-action@28d71544de8eaf1b958d335707167c5f783590ad # v0.22.2 with: - fetch-depth: 0 + image: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }} + format: cyclonedx-json + output-file: sbom.cdx.json + # ── GitHub Release ──────────────────────────────────────────────── - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - tag_name: ${{ needs.prepare-release.outputs.version_tag }} - target_commitish: ${{ needs.prepare-release.outputs.source_ref }} - name: Release ${{ needs.prepare-release.outputs.version_tag }} - body: | - Curated summary: [CHANGELOG.md](https://github.com/${{ github.repository }}/blob/${{ needs.prepare-release.outputs.version_tag }}/CHANGELOG.md) - generate_release_notes: true - draft: false - prerelease: false + if: startsWith(github.ref, 'refs/tags/') env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create "${{ github.ref_name }}" \ + --title "Release ${{ github.ref_name }}" \ + --generate-notes \ + sbom.cdx.json From b9c8645f7d62c08db090728202a45b1f90525186 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 21 Feb 2026 16:43:08 -0600 Subject: [PATCH 04/17] ci: add Dependabot for GitHub Actions updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Weekly grouped updates for CI action SHAs only. No pip ecosystem — upstream manages Python deps. Co-Authored-By: Claude Opus 4.6 --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..a078769d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Dependabot — GitHub Actions only. +# Python deps are managed by upstream; we stay in sync via git merge. +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + groups: + ci-actions: + patterns: + - "*" From f7b18f40c79fd0d2d618b6a4ce9fe1d0c10ecd48 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 21 Feb 2026 17:22:45 -0600 Subject: [PATCH 05/17] docs: add fork notice with delta from upstream Document what Fieldnote-Echo fork adds vs remsky/Kokoro-FastAPI: markdown normalization, supply-chain hardened CI, GHCR images. Co-Authored-By: Claude Opus 4.6 --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index e46a04b5..0f1b9856 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ +> **Fieldnote-Echo fork** of [remsky/Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) (currently based on `v0.6.0`) +> +> **What this fork adds:** +> - **Markdown-aware TTS normalization** — strips `**bold**`, `[links](url)`, headings, code blocks, tables, etc. before synthesis so LLM output sounds natural (toggle: `markdown_normalization` in `NormalizationOptions`, default on) +> - **Supply-chain hardened CI** — CUDA base images pinned by digest (in `docker-bake.hcl` and the release workflow), all GitHub Actions pinned to SHA, Sigstore build provenance, Trivy container scan, CycloneDX SBOM on every release +> - **GHCR images** — `ghcr.io/fieldnote-echo/kokoro-fastapi-gpu` (GPU, amd64) +> +> Everything else is upstream. We rebase on `remsky/master` periodically. +

Kokoro TTS Banner

From 780f689b399d986d68156725a5596fcb6f3e3758 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:43:13 -0500 Subject: [PATCH 06/17] test(md): relocate markdown normalizer tests into api/tests Move api/src/services/text_processing/test_normalizer.py to api/tests/test_markdown_normalizer.py so pytest's testpaths collects it, and replace the import shim with a direct package import. Co-Authored-By: Claude Fable 5 --- .../test_markdown_normalizer.py} | 69 +------------------ 1 file changed, 3 insertions(+), 66 deletions(-) rename api/{src/services/text_processing/test_normalizer.py => tests/test_markdown_normalizer.py} (64%) diff --git a/api/src/services/text_processing/test_normalizer.py b/api/tests/test_markdown_normalizer.py similarity index 64% rename from api/src/services/text_processing/test_normalizer.py rename to api/tests/test_markdown_normalizer.py index 2478b2ee..a47eafc8 100644 --- a/api/src/services/text_processing/test_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -1,69 +1,6 @@ -"""Unit tests for handle_markdown() — the navi-os fork of Kokoro's normalizer. - -Tests run standalone: pytest deploy/desktop/kokoro/test_normalizer.py -v -No Docker or Kokoro dependencies required — we import handle_markdown directly. -""" - -import sys -import types -from pathlib import Path -from unittest.mock import MagicMock - -# --------------------------------------------------------------------------- -# Shim: fake Kokoro's third-party and internal dependencies so we can import -# normalizer.py without the full Kokoro environment. We only need -# handle_markdown(), which uses nothing beyond the stdlib `re` module. -# --------------------------------------------------------------------------- - -# Third-party libs used at module level in normalizer.py -for _dep in ("inflect", "numpy", "numpy.number", "torch"): - if _dep not in sys.modules: - sys.modules[_dep] = MagicMock() - -# Internal Kokoro package hierarchy for the relative import: -# from ...structures.schemas import NormalizationOptions -_fake_schemas = types.ModuleType("api.src.structures.schemas") - - -class _StubNormalizationOptions: - pass - - -_fake_schemas.NormalizationOptions = _StubNormalizationOptions # type: ignore[attr-defined] - -for mod_name in [ - "api", - "api.src", - "api.src.services", - "api.src.services.text_processing", - "api.src.structures", -]: - if mod_name not in sys.modules: - sys.modules[mod_name] = types.ModuleType(mod_name) - -sys.modules["api.src.structures.schemas"] = _fake_schemas - -# Wire up sub-module attributes so dotted access works -sys.modules["api"].src = sys.modules["api.src"] # type: ignore[attr-defined] -sys.modules["api.src"].services = sys.modules["api.src.services"] # type: ignore[attr-defined] -sys.modules["api.src"].structures = sys.modules["api.src.structures"] # type: ignore[attr-defined] -sys.modules["api.src.services"].text_processing = sys.modules["api.src.services.text_processing"] # type: ignore[attr-defined] -sys.modules["api.src.structures"].schemas = _fake_schemas # type: ignore[attr-defined] - -# Import normalizer with correct __package__ for relative imports -import importlib.util - -_spec = importlib.util.spec_from_file_location( - "api.src.services.text_processing.normalizer", - Path(__file__).resolve().parent / "normalizer.py", - submodule_search_locations=[], -) -_mod = importlib.util.module_from_spec(_spec) # type: ignore[arg-type] -_mod.__package__ = "api.src.services.text_processing" # type: ignore[attr-defined] -sys.modules[_spec.name] = _mod # type: ignore[union-attr] -_spec.loader.exec_module(_mod) # type: ignore[union-attr] - -handle_markdown = _mod.handle_markdown # type: ignore[union-attr] +"""Unit tests for handle_markdown() — the navi-os fork of Kokoro's normalizer.""" + +from api.src.services.text_processing.normalizer import handle_markdown # ── Tests ──────────────────────────────────────────────────────────────── From e727c375dcc900f1cfe09086e8569a7397478152 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:47:56 -0500 Subject: [PATCH 07/17] fix(md): let emphasis span soft-wrap newlines but not blank lines _MD_BOLD/_MD_ITALIC_STAR/_MD_ITALIC_UNDER/_MD_STRIKETHROUGH used .+? without newline handling, so '**bold\ntext**' passed through unstripped. Emphasis content now matches across single newlines (soft wraps) while refusing to cross blank lines (paragraph breaks), keeping over-matching across paragraphs impossible. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 11 ++++--- api/tests/test_markdown_normalizer.py | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 8f01830f..aa236b37 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -200,13 +200,16 @@ _MD_INLINE_CODE = re.compile(r"`([^`]+)`") _MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]+\)") _MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)") -_MD_BOLD = re.compile(r"\*\*(.+?)\*\*") +# Emphasis content: spans single newlines (soft wraps) but never a blank +# line (paragraph break) — CommonMark-ish, avoids catastrophic over-matching. +_MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n))+?" +_MD_BOLD = re.compile(r"\*\*(" + _MD_EMPHASIS_CONTENT + r")\*\*") # Italic with * — only match when not preceded/followed by word chars to avoid # false positives inside URLs or filenames. -_MD_ITALIC_STAR = re.compile(r"(?\s?", re.MULTILINE) _MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index a47eafc8..bf392ce9 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -159,3 +159,34 @@ def test_bold_with_surrounding_text(): def test_link_with_surrounding_text(): result = handle_markdown("Click [here](https://x.com) now") assert result == "Click here now" + + +# ── Multiline emphasis (soft wraps) ────────────────────────────────────── + + +def test_bold_across_soft_wrap(): + """Emphasis should span a single newline (soft-wrapped paragraph).""" + assert handle_markdown("**bold\ntext**") == "bold\ntext" + + +def test_italic_star_across_soft_wrap(): + assert handle_markdown("*ital\nic*") == "ital\nic" + + +def test_italic_underscore_across_soft_wrap(): + assert handle_markdown("_ital\nic_") == "ital\nic" + + +def test_strikethrough_across_soft_wrap(): + assert handle_markdown("~~gone\nnow~~") == "gone\nnow" + + +def test_emphasis_does_not_span_blank_lines(): + """A blank line is a paragraph break — emphasis must not match across it.""" + md = "**alpha\n\nbeta**" + assert handle_markdown(md) == md + + +def test_emphasis_does_not_span_whitespace_only_blank_lines(): + md = "**alpha\n \nbeta**" + assert handle_markdown(md) == md From f7a727fc1904b727db9b92e4f06b5d06f565ab9f Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:48:35 -0500 Subject: [PATCH 08/17] fix(md): treat unclosed code fences as extending to end of input Streaming/truncated chunks can open a ``` fence that never closes; the old pattern required a closing fence, leaving literal backticks that got voiced. The fence must now start at a line beginning, and an unclosed fence swallows to end of input: the fence line (with its language info string) is stripped and the content is kept protected. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 6 +++- api/tests/test_markdown_normalizer.py | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index aa236b37..ca524bad 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -196,7 +196,11 @@ # --------------------------------------------------------------------------- # Order matters: fenced code blocks first, then inline, then structural. # Bold before italic to avoid partial matches on **. -_MD_FENCED_CODE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) +# A fence opens at line start; an unclosed fence (streaming/truncated +# chunks) extends to end of input — strip the fence line, keep the content. +_MD_FENCED_CODE = re.compile( + r"^[ \t]*```[^\n]*\n?(.*?)(?:```|\Z)", re.DOTALL | re.MULTILINE +) _MD_INLINE_CODE = re.compile(r"`([^`]+)`") _MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]+\)") _MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)") diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index bf392ce9..a2078e69 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -190,3 +190,31 @@ def test_emphasis_does_not_span_blank_lines(): def test_emphasis_does_not_span_whitespace_only_blank_lines(): md = "**alpha\n \nbeta**" assert handle_markdown(md) == md + + +# ── Unclosed fenced code blocks (streaming / truncated chunks) ─────────── + + +def test_unclosed_fenced_code_block(): + """An unclosed fence extends to end of input: fence line stripped, content kept.""" + result = handle_markdown("```python\nprint('hi')") + assert result == "print('hi')" + + +def test_unclosed_fenced_code_block_with_prefix(): + result = handle_markdown("Here is code:\n```python\nmodel.train()\n") + assert "`" not in result + assert "python" not in result + assert "model.train()" in result + assert "Here is code:" in result + + +def test_unclosed_fence_protects_content(): + """Markdown inside an unclosed fence must not be stripped.""" + result = handle_markdown("```\n**not stripped**") + assert "**not stripped**" in result + + +def test_bare_unclosed_fence_line(): + """A lone opening fence with no content voices nothing.""" + assert handle_markdown("```python").strip() == "" From 7e287393df1a3e1dddac491a361e720a0802627e Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:49:18 -0500 Subject: [PATCH 09/17] fix(md): tolerate balanced parens in link and image URLs _MD_LINK/_MD_IMAGE stopped at the first ')', so links to URLs like wiki/Foo_(bar) left a stray ')' in the voiced text. The URL part now accepts one nesting level of balanced parentheses, which covers real-world URLs while still stopping at the true closing paren. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 7 ++++-- api/tests/test_markdown_normalizer.py | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index ca524bad..11ab1d3b 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -202,8 +202,11 @@ r"^[ \t]*```[^\n]*\n?(.*?)(?:```|\Z)", re.DOTALL | re.MULTILINE ) _MD_INLINE_CODE = re.compile(r"`([^`]+)`") -_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\([^)]+\)") -_MD_LINK = re.compile(r"\[([^\]]+)\]\([^)]+\)") +# URL part tolerates one level of balanced parens, e.g. wiki/Foo_(bar) — +# enough for real-world URLs without over-matching past the closing ). +_MD_URL_PART = r"(?:[^()]|\([^()]*\))+" +_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") +_MD_LINK = re.compile(r"\[([^\]]+)\]\(" + _MD_URL_PART + r"\)") # Emphasis content: spans single newlines (soft wraps) but never a blank # line (paragraph break) — CommonMark-ish, avoids catastrophic over-matching. _MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n))+?" diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index a2078e69..c493daf8 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -218,3 +218,27 @@ def test_unclosed_fence_protects_content(): def test_bare_unclosed_fence_line(): """A lone opening fence with no content voices nothing.""" assert handle_markdown("```python").strip() == "" + + +# ── Parenthesized URLs in links and images ─────────────────────────────── + + +def test_link_with_parenthesized_url(): + md = "[Foo](https://en.wikipedia.org/wiki/Foo_(bar))" + assert handle_markdown(md) == "Foo" + + +def test_image_with_parenthesized_url(): + md = "![alt text](https://example.com/img_(1).png)" + assert handle_markdown(md) == "alt text" + + +def test_link_paren_url_with_surrounding_text(): + md = "See [Foo](https://en.wikipedia.org/wiki/Foo_(bar)) today" + assert handle_markdown(md) == "See Foo today" + + +def test_link_plain_url_still_stops_at_paren(): + """A plain link followed by a parenthetical must not over-match.""" + md = "[here](https://x.com) (see note)" + assert handle_markdown(md) == "here (see note)" From 733f74fd2da656ac84697ed1b73afc4f4824b5ad Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:50:14 -0500 Subject: [PATCH 10/17] fix(md): preserve segment boundary spaces around custom phonemes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smart_split splits on CUSTOM_PHONEMES and rejoins with ''.join, so a normalizer that strips segment edges glues words to phoneme tokens ('Say[Kokoro](...)please'). Restore any edge whitespace the normalizer eats and strip once on the final joined result only. Note: the rebase onto upstream already dropped normalize_text's trailing .strip(), so this did not reproduce here — this hardens the join and adds the phoneme-spacing regression tests so it cannot come back. Co-Authored-By: Claude Fable 5 --- .../text_processing/text_processor.py | 16 +++++-- api/tests/test_markdown_normalizer.py | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/api/src/services/text_processing/text_processor.py b/api/src/services/text_processing/text_processor.py index cce7a128..cdfc1ee0 100644 --- a/api/src/services/text_processing/text_processor.py +++ b/api/src/services/text_processing/text_processor.py @@ -173,10 +173,18 @@ async def smart_split( if lang_code in ["a", "b", "en-us", "en-gb"]: processed_text = CUSTOM_PHONEMES.split(processed_text) for index in range(0, len(processed_text), 2): - processed_text[index] = normalize_text( - processed_text[index], normalization_options - ) - + segment = processed_text[index] + normalized = normalize_text(segment, normalization_options) + # Segments are rejoined with ''.join below, so any edge + # whitespace the normalizer eats would glue words onto + # the adjacent custom phoneme tokens — restore it. + if segment[:1].isspace() and not normalized[:1].isspace(): + normalized = " " + normalized + if segment[-1:].isspace() and not normalized[-1:].isspace(): + normalized = normalized + " " + processed_text[index] = normalized + + # Strip once on the final joined result, never per segment processed_text = "".join(processed_text).strip() else: logger.info( diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index c493daf8..5844b236 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -1,6 +1,10 @@ """Unit tests for handle_markdown() — the navi-os fork of Kokoro's normalizer.""" +import pytest + from api.src.services.text_processing.normalizer import handle_markdown +from api.src.services.text_processing.text_processor import smart_split +from api.src.structures.schemas import NormalizationOptions # ── Tests ──────────────────────────────────────────────────────────────── @@ -242,3 +246,42 @@ def test_link_plain_url_still_stops_at_paren(): """A plain link followed by a parenthetical must not over-match.""" md = "[here](https://x.com) (see note)" assert handle_markdown(md) == "here (see note)" + + +# ── Custom phoneme spacing through smart_split ─────────────────────────── + + +@pytest.mark.asyncio +async def test_custom_phoneme_spacing_preserved(): + """Words must not get glued to custom phoneme tokens during normalization. + + smart_split splits on CUSTOM_PHONEMES and rejoins segments with ''.join; + if segment edges get stripped, 'Say [Kokoro](...) please' becomes + 'Say[Kokoro](...)please'. + """ + chunks = [] + async for chunk_text, _, _ in smart_split( + "Say [Kokoro](/kˈOkəɹO/) please.", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "Say [Kokoro](/kˈOkəɹO/) please." in combined + assert "Say[" not in combined + assert ")please" not in combined + + +@pytest.mark.asyncio +async def test_custom_phoneme_spacing_preserved_between_tokens(): + """A whitespace-only segment between two phoneme tokens must survive.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "[Kokoro](/kˈOkəɹO/) [rest](/ɹˈɛst/) now.", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "/) [rest]" in combined + assert "/)[rest]" not in combined From 551488d6ef9c4f701b3cb63b609ae25b1bd6b9a1 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:51:01 -0500 Subject: [PATCH 11/17] fix(md): apply markdown normalization for all languages smart_split only ran normalize_text for lang codes a/b/en-us/en-gb, so markdown_normalization was silently ignored for every other language. Markdown syntax is language-independent: when the toggle is enabled, handle_markdown() now runs for non-English languages too (with custom phoneme tokens protected), while the rest of the English-specific normalization stays gated as before. Co-Authored-By: Claude Fable 5 --- .../text_processing/text_processor.py | 17 +++++- api/tests/test_markdown_normalizer.py | 55 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/api/src/services/text_processing/text_processor.py b/api/src/services/text_processing/text_processor.py index cdfc1ee0..727e5056 100644 --- a/api/src/services/text_processing/text_processor.py +++ b/api/src/services/text_processing/text_processor.py @@ -8,7 +8,7 @@ from ...core.config import settings from ...structures.schemas import NormalizationOptions -from .normalizer import normalize_text +from .normalizer import handle_markdown, normalize_text from .phonemizer import phonemize from .vocabulary import tokenize @@ -186,6 +186,21 @@ async def smart_split( # Strip once on the final joined result, never per segment processed_text = "".join(processed_text).strip() + elif normalization_options.markdown_normalization: + # Markdown syntax is language-independent — strip it even + # when English-specific normalization is skipped. Custom + # phoneme tokens look like markdown links, so protect them. + processed_text = CUSTOM_PHONEMES.split(processed_text) + for index in range(0, len(processed_text), 2): + processed_text[index] = handle_markdown( + processed_text[index] + ) + + processed_text = "".join(processed_text).strip() + logger.info( + "Applied markdown normalization only; full text " + "normalization is only supported for english" + ) else: logger.info( "Skipping text normalization as it is only supported for english" diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index 5844b236..be1a96b4 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -285,3 +285,58 @@ async def test_custom_phoneme_spacing_preserved_between_tokens(): combined = " ".join(chunks) assert "/) [rest]" in combined assert "/)[rest]" not in combined + + +# ── Markdown normalization for non-English languages ───────────────────── + + +@pytest.mark.asyncio +async def test_markdown_stripped_for_non_english_language(): + """Markdown syntax is language-independent — it must be stripped even + when the rest of (English-specific) normalization is skipped.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "# Heading\nDas ist **fett** und [Link](https://example.com).", + lang_code="e", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "**" not in combined + assert "#" not in combined + assert "](" not in combined + assert "fett" in combined + assert "Link" in combined + + +@pytest.mark.asyncio +async def test_non_english_markdown_keeps_custom_phonemes(): + """Custom phoneme tokens look like markdown links — they must survive + the markdown pass for non-English languages too.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "Sag [Kokoro](/kˈOkəɹO/) **bitte**.", + lang_code="e", + normalization_options=NormalizationOptions(), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "[Kokoro](/kˈOkəɹO/)" in combined + assert "**" not in combined + + +@pytest.mark.asyncio +async def test_non_english_markdown_respects_toggle(): + """With markdown_normalization disabled, non-English text is untouched.""" + chunks = [] + async for chunk_text, _, _ in smart_split( + "Das ist **fett**.", + lang_code="e", + normalization_options=NormalizationOptions(markdown_normalization=False), + ): + chunks.append(chunk_text) + + combined = " ".join(chunks) + assert "**fett**" in combined From 84c7e9a4a70cba259f768471533342a9ee83fa44 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 10:53:05 -0500 Subject: [PATCH 12/17] fix(md): batch of minor markdown normalization fixes - __bold__ double-underscore emphasis is now stripped (word-boundary aware, like the underscore italic) - ATX-closed headings drop the trailing hash sequence ('## H ##' -> 'H') while '#' glued to a word survives ('# Learning C#' -> 'Learning C#') - list markers are stripped before headings so '- # H' voices 'H'; heading regex also tolerates indentation - empty-link-text '[](url)' drops entirely instead of surviving verbatim - NUL bytes are removed from input up front, making the \x00-delimited code placeholders collision-proof against placeholder-looking input - table rows still need 2+ pipes but must also be pipe-anchored or sit next to a separator row, so prose like 'a | b | c' keeps its pipes Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 51 +++++++++++---- api/tests/test_markdown_normalizer.py | 63 +++++++++++++++++++ 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 11ab1d3b..9666a2e7 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -206,18 +206,25 @@ # enough for real-world URLs without over-matching past the closing ). _MD_URL_PART = r"(?:[^()]|\([^()]*\))+" _MD_IMAGE = re.compile(r"!\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") -_MD_LINK = re.compile(r"\[([^\]]+)\]\(" + _MD_URL_PART + r"\)") +# Link text may be empty ('[](url)') — the whole link then drops entirely. +_MD_LINK = re.compile(r"\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") # Emphasis content: spans single newlines (soft wraps) but never a blank # line (paragraph break) — CommonMark-ish, avoids catastrophic over-matching. _MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n))+?" _MD_BOLD = re.compile(r"\*\*(" + _MD_EMPHASIS_CONTENT + r")\*\*") +# Bold with __ — word-boundary-aware like the underscore italic below. +_MD_BOLD_UNDER = re.compile(r"(? 'H') — but '#' glued to a word stays ('C#'). +_MD_HEADING = re.compile( + r"^[ \t]*#{1,6}[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$", re.MULTILINE +) _MD_BLOCKQUOTE = re.compile(r"^>\s?", re.MULTILINE) _MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) _MD_UNORDERED_LIST = re.compile(r"^(\s*)[-*+]\s+", re.MULTILINE) @@ -234,6 +241,11 @@ def handle_markdown(text: str) -> str: before the URL handler sees bare URLs. """ # --- Phase 1: Protect code content from markdown stripping --- + # Drop any NUL bytes first: they can't be voiced anyway, and it makes the + # \x00-delimited placeholders below collision-proof against input that + # happens to contain literal placeholder-looking text. + text = text.replace("\x00", "") + # Extract fenced code blocks and inline code into placeholders so that # markdown syntax inside code (e.g. **bold**) is preserved literally. code_blocks: list[str] = [] @@ -257,29 +269,42 @@ def _save_inline(m: re.Match) -> str: text = _MD_IMAGE.sub(r"\1", text) # Links — keep link text text = _MD_LINK.sub(r"\1", text) - # Bold (** before * to avoid partial match) + # Bold (** and __ before single-char markers to avoid partial match) text = _MD_BOLD.sub(r"\1", text) + text = _MD_BOLD_UNDER.sub(r"\1", text) # Italic (* and _) text = _MD_ITALIC_STAR.sub(r"\1", text) text = _MD_ITALIC_UNDER.sub(r"\1", text) # Strikethrough text = _MD_STRIKETHROUGH.sub(r"\1", text) - # Headings — strip leading hashes - text = _MD_HEADING.sub("", text) + # List markers — strip marker, keep text. Must run before headings so + # '- # Heading' loses both the marker and the hashes. + text = _MD_UNORDERED_LIST.sub(r"\1", text) + text = _MD_ORDERED_LIST.sub(r"\1", text) + # Headings — strip leading hashes (and any closing hash sequence) + text = _MD_HEADING.sub(r"\1", text) # Blockquotes — strip leading > text = _MD_BLOCKQUOTE.sub("", text) # Horizontal rules — remove entire line text = _MD_HORIZONTAL_RULE.sub("", text) - # List markers — strip marker, keep text - text = _MD_UNORDERED_LIST.sub(r"\1", text) - text = _MD_ORDERED_LIST.sub(r"\1", text) - # Table separator rows — remove entirely - text = _MD_TABLE_SEP_ROW.sub("", text) - # Table pipes — only replace on lines that look like table rows (2+ pipes). - # This preserves non-table pipes like "true | false". + # Tables — remove separator rows, replace pipes with spaces on table + # rows. A row must have 2+ pipes AND either be pipe-anchored (leading or + # trailing |) or sit next to a separator row; prose pipes like + # "either a | b | c works" stay intact. lines = text.split("\n") + is_sep_row = [bool(_MD_TABLE_SEP_ROW.match(line)) for line in lines] for i, line in enumerate(lines): - if line.count("|") >= 2: + if is_sep_row[i]: + lines[i] = "" + continue + if line.count("|") < 2: + continue + stripped = line.strip() + pipe_anchored = stripped.startswith("|") or stripped.endswith("|") + near_sep_row = (i > 0 and is_sep_row[i - 1]) or ( + i + 1 < len(lines) and is_sep_row[i + 1] + ) + if pipe_anchored or near_sep_row: lines[i] = line.replace("|", " ") text = "\n".join(lines) diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index be1a96b4..49bff45e 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -340,3 +340,66 @@ async def test_non_english_markdown_respects_toggle(): combined = " ".join(chunks) assert "**fett**" in combined + + +# ── Minor batch: emphasis, headings, links, sentinels, tables ──────────── + + +def test_bold_double_underscore(): + assert handle_markdown("__bold__") == "bold" + + +def test_bold_double_underscore_with_surrounding_text(): + assert handle_markdown("This is __bold__ text") == "This is bold text" + + +def test_closed_atx_heading(): + """ATX headings may close with trailing hashes: '## H ##' -> 'H'.""" + assert handle_markdown("## Heading ##") == "Heading" + + +def test_heading_with_trailing_hash_word_kept(): + """A '#' glued to a word is content, not a closing sequence.""" + assert handle_markdown("# Learning C#") == "Learning C#" + + +def test_heading_after_list_marker(): + """List markers are stripped before headings so '- # H' voices 'H'.""" + assert handle_markdown("- # Heading") == "Heading" + + +def test_heading_after_ordered_list_marker(): + assert handle_markdown("1. ## Heading") == "Heading" + + +def test_empty_link_text_dropped(): + result = handle_markdown("before [](https://example.com) after") + assert "[" not in result + assert "]" not in result + assert "example.com" not in result + assert "before" in result + assert "after" in result + + +def test_sentinel_collision_safe(): + """Literal placeholder-looking input must not corrupt code restoration.""" + md = "weird \x00CB0\x00 input\n```\nsecret code\n```" + result = handle_markdown(md) + assert result.count("secret code") == 1 + + +def test_prose_with_pipes_not_table(): + """2+ pipes in prose without table shape must be preserved.""" + text = "either a | b | c works" + assert handle_markdown(text) == text + + +def test_headerless_table_rows_near_separator(): + """Rows without leading/trailing pipes count as table when adjacent to a + separator row.""" + md = "Name | Age | City\n--- | --- | ---\nAlice | 30 | NYC" + result = handle_markdown(md) + assert "|" not in result + assert "---" not in result + assert "Alice" in result + assert "NYC" in result From 50c863a1654a243e126eb20dfed383512dcd0614 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 11:04:29 -0500 Subject: [PATCH 13/17] fix(md): normalize CRLF, bound emphasis scan, blockquoted fences, escaped pipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifier findings on the md normalization pass: - CRLF input defeated every MULTILINE anchor: closing-hash headings leaked '##' into speech and CRLF blank lines stopped acting as paragraph breaks for emphasis. Normalize \r\n/\r to \n up front. - Emphasis patterns scanned O(n^2) on floods of unclosed markers (140KB of '__word ' took 43s inside async smart_split — an event-loop-blocking single-request DoS). Bound emphasis spans at 600 chars: O(n·600), same 140KB now 0.34s. - Fences inside blockquotes were garbled by the inline-code pass; recognize '> ```' openers and dedent the quoted content. - Escaped pipes (\|) were counted as table syntax; treat as literal. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 31 +++++++-- api/tests/test_markdown_normalizer.py | 66 +++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 9666a2e7..174265a0 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -196,11 +196,15 @@ # --------------------------------------------------------------------------- # Order matters: fenced code blocks first, then inline, then structural. # Bold before italic to avoid partial matches on **. -# A fence opens at line start; an unclosed fence (streaming/truncated -# chunks) extends to end of input — strip the fence line, keep the content. +# A fence opens at line start (optionally inside a blockquote); an unclosed +# fence (streaming/truncated chunks) extends to end of input — strip the +# fence line, keep the content. _MD_FENCED_CODE = re.compile( - r"^[ \t]*```[^\n]*\n?(.*?)(?:```|\Z)", re.DOTALL | re.MULTILINE + r"^[ \t]*(>[ \t]*)?```[^\n]*\n?(.*?)(?:```|\Z)", re.DOTALL | re.MULTILINE ) +# Blockquote marker to dedent from fenced content when the fence itself was +# blockquoted ('> ```' ... '> code' ... '> ```'). +_MD_BLOCKQUOTE_DEDENT = re.compile(r"^>[ \t]?", re.MULTILINE) _MD_INLINE_CODE = re.compile(r"`([^`]+)`") # URL part tolerates one level of balanced parens, e.g. wiki/Foo_(bar) — # enough for real-world URLs without over-matching past the closing ). @@ -209,8 +213,10 @@ # Link text may be empty ('[](url)') — the whole link then drops entirely. _MD_LINK = re.compile(r"\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") # Emphasis content: spans single newlines (soft wraps) but never a blank -# line (paragraph break) — CommonMark-ish, avoids catastrophic over-matching. -_MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n))+?" +# line (paragraph break) — CommonMark-ish. Bounded at 600 chars so a flood +# of unclosed markers scans O(n·600) instead of O(n^2); longer "spans" are +# not real emphasis and simply stay unstripped. +_MD_EMPHASIS_CONTENT = r"(?:[^\n]|\n(?![ \t]*\n)){1,600}?" _MD_BOLD = re.compile(r"\*\*(" + _MD_EMPHASIS_CONTENT + r")\*\*") # Bold with __ — word-boundary-aware like the underscore italic below. _MD_BOLD_UNDER = re.compile(r"(? str: # \x00-delimited placeholders below collision-proof against input that # happens to contain literal placeholder-looking text. text = text.replace("\x00", "") + # Normalize line endings: every MULTILINE anchor and blank-line lookahead + # below assumes \n; raw \r defeats them (CRLF headings leak '##', CRLF + # blank lines stop acting as paragraph breaks for emphasis). + text = text.replace("\r\n", "\n").replace("\r", "\n") # Extract fenced code blocks and inline code into placeholders so that # markdown syntax inside code (e.g. **bold**) is preserved literally. code_blocks: list[str] = [] def _save_fenced(m: re.Match) -> str: - code_blocks.append(m.group(1)) + content = m.group(2) + if m.group(1): + # Fence was blockquoted — dedent the '> ' markers from content. + content = _MD_BLOCKQUOTE_DEDENT.sub("", content) + code_blocks.append(content) return f"\x00CB{len(code_blocks) - 1}\x00" text = _MD_FENCED_CODE.sub(_save_fenced, text) @@ -291,6 +305,9 @@ def _save_inline(m: re.Match) -> str: # rows. A row must have 2+ pipes AND either be pipe-anchored (leading or # trailing |) or sit next to a separator row; prose pipes like # "either a | b | c works" stay intact. + # Escaped pipes (\|) are literal content — hide them from the table + # logic, then restore as bare pipes at the end. + text = text.replace("\\|", "\x00EP\x00") lines = text.split("\n") is_sep_row = [bool(_MD_TABLE_SEP_ROW.match(line)) for line in lines] for i, line in enumerate(lines): @@ -306,7 +323,7 @@ def _save_inline(m: re.Match) -> str: ) if pipe_anchored or near_sep_row: lines[i] = line.replace("|", " ") - text = "\n".join(lines) + text = "\n".join(lines).replace("\x00EP\x00", "|") # --- Phase 3: Restore protected code content --- for idx, content in enumerate(code_blocks): diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index 49bff45e..c07bc400 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -403,3 +403,69 @@ def test_headerless_table_rows_near_separator(): assert "---" not in result assert "Alice" in result assert "NYC" in result + + +# --------------------------------------------------------------------------- +# CRLF handling (verifier findings: MULTILINE anchors assume \n) +# --------------------------------------------------------------------------- + + +def test_crlf_closed_atx_heading(): + """Closing-hash headings must strip on CRLF input, not leak '##'.""" + result = handle_markdown("## Heading ##\r\nbody") + assert "#" not in result + assert "Heading" in result + assert "body" in result + + +def test_crlf_blank_line_stops_emphasis(): + """A CRLF blank line is a paragraph break — emphasis must not span it.""" + result = handle_markdown("*start\r\n\r\nend*") + assert "*start" in result + assert "end*" in result + + +def test_crlf_equivalent_to_lf(): + """CRLF input must normalize to the same speech text as LF input.""" + lf = handle_markdown("# Title\n\n**bold** and _em_\n\n- item") + crlf = handle_markdown("# Title\r\n\r\n**bold** and _em_\r\n\r\n- item") + assert crlf == lf + + +# --------------------------------------------------------------------------- +# Pathological-input performance (verifier finding: quadratic emphasis scan) +# --------------------------------------------------------------------------- + + +def test_unclosed_emphasis_flood_is_fast(): + """Repeated unclosed emphasis markers must not scan quadratically. + + 100KB of '*word ' previously took >10s (O(n^2)); bounded emphasis + spans make it linear. Generous ceiling to avoid CI flakiness. + """ + import time + + for marker in ("*word ", "_word ", "__word "): + payload = marker * (100_000 // len(marker)) + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 2.0 + + +# --------------------------------------------------------------------------- +# Blockquoted fences and escaped pipes (verifier minors) +# --------------------------------------------------------------------------- + + +def test_fenced_code_inside_blockquote(): + """A fence opened inside a blockquote must not be garbled by the + inline-code pass pairing backticks across lines.""" + result = handle_markdown("> ```\n> code line\n> ```") + assert "`" not in result + assert "code line" in result + + +def test_escaped_pipes_not_table(): + """Escaped pipes are literal content, not table syntax.""" + result = handle_markdown("use a \\| b \\| c here") + assert result == "use a | b | c here" From ff9dc91ebc7c087b045570a49ec54b627dd8bce1 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 11:28:39 -0500 Subject: [PATCH 14/17] fix(md): eliminate remaining quadratic scans in markdown pass Round-2 verifier profiling found three more super-linear vectors on the same unbounded request path as the emphasis fix: - _MD_TABLE_SEP_ROW: '-{3,}' and '[\s:|-]*' both match '-', giving catastrophic backtracking on long dash lines (80KB line took 43s). Replaced with an ambiguity-free character-class fullmatch plus a '---' substring check. - _MD_LINK/_MD_IMAGE: unbounded '[^\]]*' link text scanned to end-of-string from every '[' in a bracket flood (400KB took >30s). Bounded link text at 800 chars and the URL part at 2000. - Phase-3 placeholder restore looped str.replace per placeholder, O(placeholders x len): fence/backtick floods were quadratic even with fast regexes. Replaced with a single indexed-callback re.sub. All vectors now complete in <0.25s at 400-800KB. Adds flood and scaling-ratio regression tests. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 35 +++++++++++++----- api/tests/test_markdown_normalizer.py | 37 +++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 174265a0..7a6cffc2 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -205,13 +205,19 @@ # Blockquote marker to dedent from fenced content when the fence itself was # blockquoted ('> ```' ... '> code' ... '> ```'). _MD_BLOCKQUOTE_DEDENT = re.compile(r"^>[ \t]?", re.MULTILINE) +# Placeholder restore patterns (Phase 3) — input NULs are stripped up front, +# so these can only match placeholders this module minted. +_MD_CB_PLACEHOLDER = re.compile("\x00CB(\\d+)\x00") +_MD_IC_PLACEHOLDER = re.compile("\x00IC(\\d+)\x00") _MD_INLINE_CODE = re.compile(r"`([^`]+)`") # URL part tolerates one level of balanced parens, e.g. wiki/Foo_(bar) — # enough for real-world URLs without over-matching past the closing ). -_MD_URL_PART = r"(?:[^()]|\([^()]*\))+" -_MD_IMAGE = re.compile(r"!\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") +# Link text and URL are length-bounded like emphasis: on '[' floods the +# unbounded '[^\]]*' scanned to end-of-string from every position (O(n^2)). +_MD_URL_PART = r"(?:[^()]|\([^()]*\)){1,2000}" +_MD_IMAGE = re.compile(r"!\[([^\]]{0,800})\]\(" + _MD_URL_PART + r"\)") # Link text may be empty ('[](url)') — the whole link then drops entirely. -_MD_LINK = re.compile(r"\[([^\]]*)\]\(" + _MD_URL_PART + r"\)") +_MD_LINK = re.compile(r"\[([^\]]{0,800})\]\(" + _MD_URL_PART + r"\)") # Emphasis content: spans single newlines (soft wraps) but never a blank # line (paragraph break) — CommonMark-ish. Bounded at 600 chars so a flood # of unclosed markers scans O(n·600) instead of O(n^2); longer "spans" are @@ -235,8 +241,15 @@ _MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) _MD_UNORDERED_LIST = re.compile(r"^(\s*)[-*+]\s+", re.MULTILINE) _MD_ORDERED_LIST = re.compile(r"^(\s*)\d+\.\s+", re.MULTILINE) -# Table separator rows like |---|---| -_MD_TABLE_SEP_ROW = re.compile(r"^\s*\|?[\s:]*-{3,}[\s:|-]*\|?\s*$", re.MULTILINE) +# Table separator rows like |---|---|. Detected with a single ambiguity-free +# character class plus a substring check: the old '-{3,}[\s:|-]*' form had two +# adjacent quantifiers both matching '-', giving O(n^2) backtracking on long +# dash lines. +_MD_TABLE_SEP_CHARS = re.compile(r"[ \t:|-]*") + + +def _is_table_sep_row(line: str) -> bool: + return "---" in line and _MD_TABLE_SEP_CHARS.fullmatch(line) is not None def handle_markdown(text: str) -> str: @@ -309,7 +322,7 @@ def _save_inline(m: re.Match) -> str: # logic, then restore as bare pipes at the end. text = text.replace("\\|", "\x00EP\x00") lines = text.split("\n") - is_sep_row = [bool(_MD_TABLE_SEP_ROW.match(line)) for line in lines] + is_sep_row = [_is_table_sep_row(line) for line in lines] for i, line in enumerate(lines): if is_sep_row[i]: lines[i] = "" @@ -326,10 +339,12 @@ def _save_inline(m: re.Match) -> str: text = "\n".join(lines).replace("\x00EP\x00", "|") # --- Phase 3: Restore protected code content --- - for idx, content in enumerate(code_blocks): - text = text.replace(f"\x00CB{idx}\x00", content) - for idx, content in enumerate(inline_codes): - text = text.replace(f"\x00IC{idx}\x00", content) + # Single-pass sub keyed on the placeholder index: a per-item str.replace + # loop is O(placeholders x len(text)) — quadratic on fence/backtick floods. + if code_blocks: + text = _MD_CB_PLACEHOLDER.sub(lambda m: code_blocks[int(m.group(1))], text) + if inline_codes: + text = _MD_IC_PLACEHOLDER.sub(lambda m: inline_codes[int(m.group(1))], text) return text diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index c07bc400..2fbc50d1 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -469,3 +469,40 @@ def test_escaped_pipes_not_table(): """Escaped pipes are literal content, not table syntax.""" result = handle_markdown("use a \\| b \\| c here") assert result == "use a | b | c here" + + +def test_adversarial_floods_are_fast(): + """Every markdown pass must stay near-linear on pathological input. + + Covers the verifier-found quadratic vectors beyond emphasis: long + dash lines (table separator backtracking), '[' floods (link/image + scan-to-EOS), fence-opener and backtick-pair floods (per-placeholder + str.replace restore loop). + """ + import time + + payloads = [ + "-" * 40_000 + ".", # table separator backtracking + "[word " * (100_000 // 6), # unclosed-bracket link flood + "```\n" * (100_000 // 4), # fence-opener flood -> placeholder restore + "`a`" * (100_000 // 3), # inline-code flood -> placeholder restore + ] + for payload in payloads: + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 2.0, f"slow on {payload[:20]!r}..." + + +def test_emphasis_flood_scales_linearly(): + """Scaling ratio guard: 4x input must cost ~4x time (linear), not + ~16x (quadratic). Ratio bound is generous for CI noise.""" + import time + + def cost(n: int) -> float: + payload = "__word " * (n // 7) + start = time.monotonic() + handle_markdown(payload) + return time.monotonic() - start + + small, large = cost(100_000), cost(400_000) + assert large / max(small, 1e-3) < 9.0 From db2d6992f29a6e237ea0c079c01f50384e105b16 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 11:57:01 -0500 Subject: [PATCH 15/17] fix(md): eliminate quadratic scans in list markers, URL/email, acronym handler Final adversarial-hardening pass found three more O(n^2) vectors on the default normalize_text path, all reachable with tiny inputs: - _MD_UNORDERED_LIST/_MD_ORDERED_LIST: '^(\s*)' under MULTILINE let the line anchor consume every following blank line from each line start (100KB of newlines -> 65s; the CRLF fix feeds this). '\s' -> '[ \t]'. - URL_PATTERN / EMAIL_PATTERN: unbounded domain/local runs rescanned the whole tail from every position of an 'a.a.a...' flood. Added a token-start negative lookbehind to URL and bounded both to RFC lengths. - Dotted-acronym handler '(?:[A-Za-z]\.){2,} [a-z]': unbounded '{2,}' backtracked O(n) per position on 'a.a...' floods. Bounded to {2,12} (real acronyms are short); 'U.S.A. b' handling preserved. All vectors now <0.25s at 400KB (were 12-65s). Adds newline/CRLF/word-run flood regression tests. URL_PATTERN and the acronym handler are pre-existing upstream code (v0.2.4); worth upstreaming separately. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 23 +++++++++++---- api/tests/test_markdown_normalizer.py | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 7a6cffc2..4941d7dc 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -158,11 +158,19 @@ MONEY_UNITS = {"$": ("dollar", "cent"), "£": ("pound", "pence"), "€": ("euro", "cent")} # Pre-compiled regex patterns for performance +# Local part and domain are bounded at their RFC limits (64 / 253 chars): +# unbounded runs made every word boundary in 'a.a.a...' floods scan the +# whole remaining string for '@' — O(n^2). EMAIL_PATTERN = re.compile( - r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-z]{2,}\b", re.IGNORECASE + r"\b[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-z]{2,}\b", re.IGNORECASE ) URL_PATTERN = re.compile( - r"(https?://|www\.|)+(localhost|[a-zA-Z0-9.-]+(\.(?:" + # Negative lookbehind: only attempt a match at token starts. Without it + # the domain branch re-scans from every position inside a long word-char + # run (O(n^2) — a 100KB run of 'a' hung the normalizer). The domain run + # is also bounded at 253 chars, the DNS name length limit. + r"(?\s?", re.MULTILINE) _MD_HORIZONTAL_RULE = re.compile(r"^(?:---+|\*\*\*+|___+)\s*$", re.MULTILINE) -_MD_UNORDERED_LIST = re.compile(r"^(\s*)[-*+]\s+", re.MULTILINE) -_MD_ORDERED_LIST = re.compile(r"^(\s*)\d+\.\s+", re.MULTILINE) +# List indentation/spacing is spaces and tabs only — '\s' would let the +# MULTILINE '^' anchor swallow every following blank line from each line +# start, which is O(n^2) on newline floods. +_MD_UNORDERED_LIST = re.compile(r"^([ \t]*)[-*+][ \t]+", re.MULTILINE) +_MD_ORDERED_LIST = re.compile(r"^([ \t]*)\d+\.[ \t]+", re.MULTILINE) # Table separator rows like |---|---|. Detected with a single ambiguity-free # character class plus a substring check: the old '-{3,}[\s:|-]*' form had two # adjacent quantifiers both matching '-', giving O(n^2) backtracking on long @@ -663,8 +674,10 @@ def normalize_text(text: str, normalization_options: NormalizationOptions) -> st text = re.sub(r"(?<=\d)S", " S", text) text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) text = re.sub(r"(?<=X')S\b", "s", text) + # Bound the acronym run (real dotted acronyms are short): an unbounded + # '{2,}' backtracks O(n) from every position of an 'a.a.a...' flood — O(n^2). text = re.sub( - r"(?:[A-Za-z]\.){2,} [a-z]", lambda m: m.group().replace(".", "-"), text + r"(?:[A-Za-z]\.){2,12} [a-z]", lambda m: m.group().replace(".", "-"), text ) text = re.sub(r"(?i)(?<=[A-Z])\.(?=[A-Z])", "-", text) diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index 2fbc50d1..05f43fac 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -506,3 +506,31 @@ def cost(n: int) -> float: small, large = cost(100_000), cost(400_000) assert large / max(small, 1e-3) < 9.0 + + +def test_newline_and_whitespace_floods_are_fast(): + """List-marker patterns must not scan quadratically on blank-line + floods ('^(\\s*)' + MULTILINE consumed all following newlines from + every line anchor; 100KB of newlines took 65s).""" + import time + + for payload in ["\n" * 100_000, "\r\n" * 50_000, " \n" * 50_000]: + start = time.monotonic() + handle_markdown(payload) + assert time.monotonic() - start < 2.0 + + +def test_long_wordrun_through_normalize_text_is_fast(): + """URL_PATTERN must not scan quadratically on long word-char runs + with no dot-TLD (100KB single line previously hung normalize_text).""" + import time + + from api.src.services.text_processing.normalizer import ( + normalize_text, + ) + from api.src.structures.schemas import NormalizationOptions + + for payload in ["a" * 100_000, "a." * 50_000, "a@" * 50_000]: + start = time.monotonic() + normalize_text(payload, NormalizationOptions()) + assert time.monotonic() - start < 2.0 From 7d2cfe0322c3c29862f24c00a1d2db5c68b77d4d Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 12:20:23 -0500 Subject: [PATCH 16/17] docs: add fork roadmap, north star, and narration strategy State the fork's direction beyond security hardening: correct, open, tunable narration (pronunciation/prosody control for e-learning TTS). - ROADMAP.md: milestones (eval harness next), good-faith upstreaming model - NORTH_STAR.md: the distilled goal and what it explicitly is not - NARRATION_STRATEGY.md: full landscape research + market niche + primitives - README: fork notice now leads with the goal and links the roadmap Co-Authored-By: Claude Fable 5 --- NARRATION_STRATEGY.md | 158 ++++++++++++++++++++++++++++++++++++++++++ NORTH_STAR.md | 98 ++++++++++++++++++++++++++ README.md | 13 +++- ROADMAP.md | 56 +++++++++++++++ 4 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 NARRATION_STRATEGY.md create mode 100644 NORTH_STAR.md create mode 100644 ROADMAP.md diff --git a/NARRATION_STRATEGY.md b/NARRATION_STRATEGY.md new file mode 100644 index 00000000..b11f975a --- /dev/null +++ b/NARRATION_STRATEGY.md @@ -0,0 +1,158 @@ +# Decision Brief: Kokoro-Based Natural Narration (and the Singing Horizon) + +*Synthesized from 5 research angles + adversarial verification. Where a verifier corrected a finding, the corrected version is what appears below — several load-bearing claims changed under scrutiny, flagged inline as **[verified]** or **[corrected]**.** + +--- + +## 1. HOW AI SINGING IS DONE — Taxonomy & Where We Sit + +There are three distinct families. They differ primarily by **what you feed in at inference time**. + +| Family | Input at inference | Training data needed | Quality ceiling | Examples | +|---|---|---|---|---| +| **SVS (score-trained)** | Symbolic music score (phoneme + pitch-ID + note-duration streams) + lyrics. **No reference audio.** | Paired score/singing corpus per voice | Highest for singing; still fighting for every 0.5 MOS | DiffSinger, VISinger2, NNSVS; commercial: Synthesizer V, ACE Studio | +| **SVC (voice conversion)** | An **existing sung performance (audio)** + target-speaker embedding. Never sees a score. | ~10 min target-voice audio | High; preserves source expressivity | RVC (MIT), so-vits-svc (AGPL-3.0) | +| **Speech-to-singing / vocoder-retune (OURS)** | A **spoken utterance** + a target melody (note pitches/durations). DSP F0 substitution. | **Zero.** No neural training at all. | Lowest of the three — structural ceiling | WORLD/STRAIGHT pipelines; Saitou & Goto 2007 | + +**Key architectural facts (verified):** +- SVS systems (DiffSinger, VISinger2, NNSVS) genuinely take a symbolic score and need **no reference audio at inference** — this is the clean distinction from SVC. **[corrected]** Caveat: their singer *timbre* is still learned from training-time audio, and a 2025-26 zero-shot SVS branch (e.g. "Everyone-Can-Sing") *reintroduces* an inference-time reference clip for timbre cloning. So "no reference audio" is a property of *classic* score-driven SVS, not the category's future. +- DiffSinger's shallow-diffusion trick (start reverse diffusion partway using a cheap aux decoder's mel prediction, not pure noise) is **[verified]** exactly as described; openvpi fork is Apache-2.0, 44.1kHz, swappable vocoders. +- **[corrected]** NNSVS's autoregressive F0 model is framed by its own paper as an *alternative to* explicit vibrato modeling ("can generate a more natural voice without explicitly modeling vibrato"), **not** a dedicated vibrato mechanism. Minor, but don't cite it as "the vibrato model." +- RVC/so-vits-svc pipeline (content encoder → F0 extractor (RMVPE) → VITS → NSF-HiFiGAN) is **[verified]**. License gap is real and matters: **RVC is MIT, so-vits-svc is AGPL-3.0** (network-use source-disclosure obligation). + +### Where OUR speech-to-singing pipeline sits — and its honest ceiling + +Our recipe (TTS word-timestamps as forced alignment → WORLD F0-flatten-to-notes + vibrato → frame-stretch to note durations, zero training) is a **modern instance of the Saitou & Goto 2007 STRAIGHT approach**, now on WORLD. That lineage is real and works. + +**The ceiling is structural, not a tuning problem [verified]:** +- **V/UV boundary errors:** WORLD estimates F0 (DIO), spectral envelope (CheapTrick), aperiodicity (D4C) as *independent* streams. They can disagree about which frames are voiced at exactly the note onsets/offsets a singing pipeline cares about → audible discontinuities. +- **Formant-vs-pitch coupling:** harmonics are physically tied to F0. Forcing a new pitch onto an envelope estimated for the *old* pitch introduces the "buzzy"/distorted artifact — a 2026 paper (arXiv:2601.10345) exists purely to *repair* pitch-shift-degraded singing, i.e. this is treated as an unsolved problem. +- **Regime limits:** STS quality degrades on notes >150ms and beyond the speaker's corpus vocal range — exactly the sustained-note, wide-interval regime narration-to-singing hits. +- **Trained SVS itself tops out ~2.85–4.0 MOS** (ground-truth-vocoded ceiling ~4.04). A retune-of-speech approach sits *below even a naive trained SVS baseline*. Expect meaningfully lower, especially on sustained notes and vibrato. + +**[corrected] One myth to drop:** the "singing F0 range ~80–3400 Hz" figure is wrong — it conflates fundamental frequency (realistically ~65–1500 Hz) with the **singing formant** (a ~2–3.4 kHz spectral resonance). Related **[corrected]** nuance: DSP retuning *can* engineer singing-specific timbre cues — Saitou 2007 explicitly hand-builds a singer's-formant boost and vibrato-synced formant AM, and rated near-real-singing in listening tests. So the honest framing is not "DSP *cannot* model singing timbre" but "DSP models it via **hand-engineered rules**, not **learned** from data — which caps flexibility on subtler cues (breathiness, register transitions, singer idiosyncrasy)." + +**Bottom line:** Our approach's *only* durable advantage is **zero singing-training-data + any TTS voice retunable**. It is the right tool for a demo/novelty/horizon feature, the wrong tool to compete on singing quality. Treat it exactly as the concept already does: **horizon, not deliverable.** + +--- + +## 2. OUR CONCEPT: BETTER / WORSE / HOPEFUL + +### Tier 1 — md→phoneme/prosody usability layer (the NEAR-TERM GOAL) + +**This is the right bet, but it is low-novelty — that's fine, novelty isn't the goal, natural LMS narration is.** + +- **WORSE / already solved by others:** The pattern of "have the upstream LLM emit inline IPA + emphasis markup before text hits the TTS" is **already shipped vendor practice** — Inworld AI's docs literally instruct developers to prompt their LLM to wrap emphasis in asterisks and replace proper nouns with IPA. ElevenLabs v3 ships bracketed Audio Tags for the same seam. LLM-generated SSML tag placement for narration is a *published, quantified* result (ICNLSP 2025: ~50% listener-preference lift on French audiobooks with Qwen). So we are not inventing a category. +- **BETTER (our actual edge):** every one of those is **closed** and solves pronunciation as a **post-hoc, per-word patch** (pronunciation dictionaries, SSML ``, phonetic respelling) **disconnected from the content-generation step**. Articulate Storyline 360 — the dominant authoring tool — *has no pronunciation dictionary at all*, forcing authors to misspell words phonetically. The whitespace: **the LLM that drafts the course script emits the phoneme/prosody markup natively, as part of drafting**, into a phoneme-native open model we control end-to-end. Nobody ships that closed loop. +- **HOPEFUL but honest — scale is partly eating this:** Amazon's BASE TTS shows that at 10K+ training hours, "emergent" prosody appears and models infer correct *contextual/segmental* prosody from raw text with **no markup**. So the "not robotic" problem is *partly* solved by just using a good model. BUT the same paper shows scale does **not** solve emotional/emphasis intent (stuck ~2.0/3 even at max scale) or paralinguistics without explicit keywords. **Implication:** for plain "read this clearly and naturally," lean on the model + light normalization; reserve explicit markup for the things scale demonstrably *doesn't* fix — jargon pronunciation, acronyms, numbers, deliberate emphasis. Don't over-engineer prosody markup where a good voice already handles it. + +### Tier 2 — phoneme-native LLM output (the BET / HORIZON) + +**Mechanism is real and researched; the specific "external LLM hand-authors a score a frozen TTS honors" variant is unproven — and Kokoro specifically is a weak substrate for it.** + +- **BETTER / real precedent:** The decomposition "LM predicts explicit prosody tokens *first*, then conditions speech generation" is proven to help (RALL-E: WER 6.3%→2.8%; ProsodyLM; BatonVoice's "conductor LLM emits textual vocal-feature plan, BatonTTS performs it" is architecturally *our exact shape*). DiffSinger's phoneme+pitch-ID+note-duration input **is** the "explicit prosody score" interface Tier-2 wants — proving our concept rediscovers an established pattern rather than inventing one. +- **WORSE / the falsifying reality:** In *every* published case the prosody tokens are predicted by **the TTS system's own internal LM**, not authored by an upstream general-purpose LLM and honored by a frozen TTS. **Nobody has shown external-authored prosody tokens honored by a frozen model.** And the field's momentum is the *opposite direction* — natural-language style prompts (Parler-TTS, PromptTTS2, InstructTTS), i.e. describe prosody in English prose, not phonetic markup. +- **Kokoro-specific constraints (heavily revised under verification):** + - Phoneme injection `[word](/ipa/)` **works [verified]** — but **breaks on very large inputs** (~40k chars: model speaks *both* the word and the phonetic hint). **Mitigation: chunk text.** This is a real operational constraint, not a blocker. + - **[corrected — important]** The earlier "stress markup is non-functional" finding is **wrong**. Issue #170 is *closed*; the maintainer confirmed stress changes work on ≥0.9.4, and a second user confirmed the effect **does occur but is subtle and sentence-dependent** ("don't expect miracles"). So Kokoro *does* respond to `[word](+1/+2/-1/-2)` stress — just weakly. This is a *usable* signal for light emphasis, not a dead end. + - **[corrected]** The "SSML request unanswered" finding is **refuted** — issue #36 got a maintainer reply (2025-02-04) pointing to markdown pronunciation syntax + Lexicon overrides as the sanctioned surfaces, and noting emotion is blocked by **training data** (never trained on sad/angry), not by missing SSML parsing. Takeaway: emotion control won't come from markup on Kokoro at all. + - **[verified]** `speed` is `Union[float, Callable[[int],float]]` — a callable gives **per-segment** rate control (not just one global float). Under-appreciated lever. + - **[verified]** Kokoro inherits StyleTTS2's internal per-phoneme ProsodyPredictor (duration/F0/energy) but it is **not exposed** as a settable input. FastSpeech2 is the cleanest open model that *does* expose inference-time pitch/energy/duration control — but only as **scalar multipliers**, not arbitrary contours. If we ever want true per-phoneme control, forking Kokoro to expose the ProsodyPredictor (FastSpeech2-style) is the architectural template. + +**Tier-2 verdict:** Genuinely interesting, genuinely unproven, and Kokoro is not the model that will validate it (StyleTTS2's learned diffusion prosody prior may *fight* injected scores rather than compose with them — an open empirical question). Keep Tier-2 as a **research spike behind the Tier-1 product**, not a roadmap commitment. What reliably works as markup across modern open TTS is **discrete trained-in tags** ([laugh], [S1]) — Orpheus/CSM/Dia — not continuous prosody values. That tells us the honorable path is discrete, trained-in vocabulary, not free-form pitch curves bolted onto a frozen model. + +--- + +## 3. MARKET NICHE — Sharpest Defensible Position + +**The niche: "Correctness-first, self-hostable course narration where the script generator and the voice speak the same phonetic language."** + +Four combined properties none of the incumbents hold together: +1. **Open-source + Apache-2.0** (Kokoro) — self-hostable, no per-minute metering, data stays in-house. +2. **Phoneme-controllable** with an explicit IPA injection port. +3. **LLM-integrated at generation time** (markup authored *during* drafting, not patched after). +4. **LMS-embedded** — sits inside the course-authoring pipeline, not a separate voiceover SaaS. + +**Who we beat, and on what axis:** + +| Competitor | We beat them on | Their advantage over us | +|---|---|---| +| **ElevenLabs / Murf / WellSaid / PlayHT** | Cost at scale (they meter $0.15–0.30/finished-min; we self-host at ~zero marginal), data residency, generation-time pronunciation vs. their post-hoc per-word patching | Raw naturalness, voice cloning, polish | +| **Synthesia** | Price (~$2.90–3.00/min — video-bottlenecked), pure-audio focus | Talking-head video (different lane) | +| **Articulate Storyline 360 / Adobe Captivate (Polly/Azure behind them)** | They have *no* (Storyline) or legacy (Captivate) pronunciation dictionary; we make jargon/acronym correctness a first-class generation feature | Incumbent LMS integration, install base | +| **Piper / Coqui XTTS (open alternatives)** | Piper pivoted to **GPL-3.0** (Oct 2025) and has no phoneme injection; Coqui's good model (XTTS-v2) is **non-commercial** weights and dropped espeak-ng | Coqui has voice cloning | + +**The defensible axis is CORRECTNESS + CONTROL + COST, not naturalness.** We will lose a blind naturalness A/B to ElevenLabs. We win on: *"technical training content where mispronounced jargon/acronyms/numbers is unacceptable, generated at LMS scale without per-minute fees, self-hosted."* Kokoro's training distribution is **long-form reading/narration [verified]** — already the right style for course VO, a poor fit for conversational competitors. Market is large and growing (AI-voice ~$5.6–7.7B in 2026 → $21–33B by 2030-32; every vendor ships a pronunciation-editor UI, proving demand) but **no one lets the content-generation step emit the markup**. That is the wedge. + +**Honest caveat:** "correctness-first" is a real but *narrow* wedge — it's a feature, not yet a moat. The moat is the closed loop (LLM + phoneme model + LMS) being *ours end-to-end* and *open*. + +--- + +## 4. BUILD PRIMITIVES & LESSONS — Actionable Modules + +### 4.1 Core data structure: the Prosody-Annotated Phoneme IR + +A single intermediate representation everything else operates on. Proposed shape — a list of tokens: + +``` +Token { + text: str # original grapheme span + phonemes: str|None # IPA override (None = let misaki G2P handle it) + stress: int # -2..+2, maps to Kokoro [word](+N) — WORKS but subtle + emphasis: bool # drives punctuation/respelling tactics, not a fake tag + break_after_ms: int # realized via punctuation/ellipsis, not SSML + source: enum # {g2p_default, lexicon, llm_authored, human_override} +} +``` +`source` provenance is load-bearing — it lets the eval harness attribute regressions and lets human overrides always win. This IR is the contract between the LLM normalizer, the lexicon, and the Kokoro adapter. + +### 4.2 Pipeline stages (each a swappable module) + +1. **Markdown/structure normalizer** — strip/interpret markdown, expand lists/headings into speakable prose, handle code blocks (spell or skip). +2. **Number/acronym/unit expander** — deterministic, rule-based FIRST (dates, currency, units, "API"→"A P I" vs "NASA"→"nassa"). This is where scale-based TTS *fails* (BASE TTS: acronyms read letter-by-letter), so it's high-value and cheap. +3. **Lexicon resolver** — see 4.4. +4. **LLM prosody/IPA annotator** — fills `phonemes`/`stress`/`emphasis` for spans not covered by lexicon. Must be prompt-engineered + post-validated (see lesson below). +5. **Kokoro adapter** — emits `[word](/ipa/)` + `[word](+N)` + punctuation, **chunks to <~40k chars**, uses the `speed` callable for per-segment pacing. +6. **(Horizon) WORLD retune module** — separate, off the critical path. + +### 4.3 Eval harness — how to A/B naturalness (build this EARLY) + +- **Objective gate first:** run TTS output back through **ASR (WER)** against the intended script. Catches the failures that actually matter for e-learning — wrong pronunciations, dropped words, the injection-doubling bug. Cheap, automatable, CI-able. +- **Pronunciation unit tests:** a fixed corpus of jargon/acronym/number cases with expected phoneme or ASR-transcription targets. Regression suite for the lexicon + expander. +- **Human A/B (MOS/preference):** pairwise (our-normalized vs. raw-Kokoro vs. ElevenLabs) on real course paragraphs. Small n, forced-choice preference is more sensitive than absolute MOS. +- **[verified] Do NOT reuse speech-MOS models for any singing output** — SingMOS-Pro shows speech-MOS models fail on singing (domain gap). Singing needs its own raters. +- Instrument by IR `source` so you can say *which stage* moved the needle. + +### 4.4 Pronunciation lexicon for jargon/acronyms + +- **Format:** term → IPA (+ optional part-of-speech/context key for homographs). Store as versioned data, not code. +- **Resolution order:** human override > domain lexicon > deterministic acronym/number rules > LLM annotation > misaki G2P default. +- **[corrected] Validate LLM-authored IPA before injection.** LLM-G2P benchmark: naive prompting = **31.6% phoneme error rate**; optimized prompting + dictionary post-processing brought Claude 3.5 Sonnet to **5.8% PER**. Dominant failure is **inconsistent symbol choice for the same sound** (silent corruption), not obvious hallucination. So: validate LLM IPA against **Kokoro/misaki's actual phoneme inventory** and reject/normalize off-vocabulary symbols. Compounding-error risk: espeak-ng (Kokoro's fallback) itself mishandles homographs and foreign words, and **there's no downstream repair once bad phonemes enter the TTS.** +- **[corrected] Consider "LLM improves the G2P, not LLM in the inference loop."** "Fast, Not Fancy" (2025) argues augmenting fast rule-based G2P with LLM-*generated training data* beats calling an LLM at inference. Cheaper, lower-latency, deterministic at runtime — a strong option for the lexicon-building phase. + +### 4.5 Lessons already learned (bank these) + +- **[corrected] Kokoro responds to stress markup — but subtly.** `[word](+1..+2/-1..-2)` works (issue #170 closed, maintainer + user confirmed) on short/less-stressed words, effect is subtle and sentence-dependent. Usable for light emphasis; don't expect strong control. +- **[corrected] Kokoro does NOT do emotion via markup — ever.** Blocked by training data (never trained on emotion categories), per maintainer. Don't build emotion control on Kokoro; it needs a different model or fine-tune. +- **[verified] Phoneme injection is reliable at normal input sizes, breaks ~40k chars → chunk.** +- **[verified] `speed` accepts a callable for per-segment pacing** — free pacing control, currently unused. +- **Punctuation, ellipses, and phonetic respelling are the *reliable* prosody levers** across neural TTS; SSML-style tags are widely ignored. Prefer orthographic tactics over markup where possible. +- **[verified] WORLD retune works but has a hard, structural ceiling** (V/UV boundary errors, formant-pitch coupling, degrades >150ms notes / beyond corpus range). Horizon feature only. And drop the bogus "80–3400Hz F0" figure. +- **Forced alignment is free** — Kokoro's word-level timestamps *are* the alignment; no aligner needed for the retune pipeline. +- **Discrete trained-in tags are the only markup that reliably works** in modern open TTS (Orpheus/CSM/Dia). If we ever want robust expressive control, the path is trained-in vocabulary, not free-form continuous markup on a frozen model. + +--- + +## 5. RECOMMENDED FIRST MOVES (ordered) + +1. **Build the ASR-based eval harness + pronunciation unit-test corpus first.** You cannot claim "less robotic" or "correct" without measurement, and this catches the injection-doubling and jargon failures immediately. Everything downstream is graded against it. *(Smallest, highest-leverage, unblocks all A/B claims.)* + +2. **Ship the deterministic normalizer + jargon/acronym/number lexicon** (pipeline stages 1–3, IR from 4.1). This is where scale-based TTS provably fails and where our correctness wedge lives — rule-based, no LLM/model risk, immediately demoable on real Oxford House course scripts. + +3. **Add the LLM IPA/prosody annotator with a validation gate** (stage 4 + 4.4). Prompt-engineer + post-validate against Kokoro's phoneme inventory; measure PER against the corpus from step 1. Gate: must beat raw-Kokoro on the eval before it ships. Keep it *narrow* — jargon/emphasis where scale doesn't help — rather than annotating everything. + +4. **Run the first blind A/B** (our-normalized vs. raw-Kokoro vs. one commercial baseline) on real course paragraphs. Decide with data whether the Tier-1 layer actually moves naturalness/correctness enough to build the product around, and *which stage* did the work. + +5. **Time-box a Tier-2 spike (research, not product):** test whether Kokoro *honors* externally-authored stress/prosody markup at all beyond the known-subtle stress effect — one afternoon, tight hypothesis, using the harness. Result feeds the "is the phoneme-native bet real on this model?" question without committing roadmap. Keep the WORLD singing pipeline entirely off the critical path as the demo/horizon artifact. + +**Guiding principle:** the near-term goal (natural, *correct* LMS narration) is achievable, defensible, and mostly *engineering*, not research. The horizon (phoneme-native LLM output, singing) is genuinely interesting but unproven and — for singing especially — capped by physics on our chosen approach. Fund the goal; spike the horizon; don't confuse them. \ No newline at end of file diff --git a/NORTH_STAR.md b/NORTH_STAR.md new file mode 100644 index 00000000..97cec11b --- /dev/null +++ b/NORTH_STAR.md @@ -0,0 +1,98 @@ +# North Star: Correct, Open, Tunable Narration + +*Distilled from the AI-singing/TTS landscape research (see `NARRATION_STRATEGY.md`). +This supersedes "singing" as the goal. Singing was the demo that taught us where +the real gap is; it stays a horizon toy, off the critical path.* + +--- + +## The one sentence + +**A pronunciation-and-prosody control layer that lets an author — or the LLM +drafting the script — guarantee *how* an open TTS says something, not just hope +it guesses right; correct by default, tunable when it isn't, and open enough to +give back.** + +## What it is + +- **Correct.** Jargon, acronyms, numbers, units, and names come out right the + first time. Wrong pronunciation in training content isn't a polish issue — it's + a correctness bug, and we treat it like one (tested, gated, regressible). +- **Tunable.** When the default is wrong, the author fixes it *once*, in a + versioned lexicon, in the same phonetic language (IPA) the model already speaks + through its G2P — not by misspelling words to trick the voice. +- **Open.** Built on Apache-2.0 Kokoro, self-hostable, no per-minute meter, data + stays in-house. And the correctness layer itself is contributed back, not hoarded. + +## What it is NOT + +- **Not singing.** Vocoder-retune singing is physics-capped (V/UV boundary errors, + formant–pitch coupling, degrades past ~150ms notes). Fun demo, permanent horizon. +- **Not a naturalness play.** We will lose a blind naturalness A/B to ElevenLabs + and that's fine. Modern large TTS already infers good prosody from raw text; we + don't out-scale them and won't try. +- **Not markup-for-its-own-sake.** Reserve explicit control for what scale + provably *fails* at — pronunciation of jargon/acronyms/numbers and deliberate + emphasis. Everything else, let the model do its job. + +## Why this is the right long-horizon goal + +The market wedge that survived adversarial scrutiny: **correctness + control + +cost, not naturalness.** Incumbents (ElevenLabs, Murf, WellSaid) patch +pronunciation *post-hoc, per word,* behind closed metered APIs. The dominant +authoring tool (Articulate Storyline) has **no pronunciation dictionary at all.** +No one lets the content-generation step emit the pronunciation/prosody markup as +part of drafting. That closed loop — LLM + phoneme-native open model + authoring +pipeline, owned end-to-end and open — is the defensible position, and it's most +valuable precisely for technical training content where a mispronounced acronym +is unacceptable. + +## The good-faith open contribution + +Kokoro today has no lexicon / pronunciation-control layer. The parts of our work +that *every* Kokoro user benefits from should go **upstream** (to Kokoro-FastAPI +or the misaki G2P project), not stay in our fork: + +| Contribute upstream (good-faith) | Keep as our product | +|---|---| +| The Prosody-Annotated Phoneme IR (a clean token contract) | LMS/authoring integration (rippling-lms) | +| A pronunciation **lexicon resolver** + versioned lexicon format | Our domain lexicon (Oxford House jargon) | +| LLM→IPA **validation gate** (reject off-inventory phonemes) | The script-generation prompt chain | +| ReDoS/normalizer hardening *(already PR'd: remsky#489)* | The closed generation→narration loop | + +The ReDoS fix we already sent upstream is the template: find a real gap in a +solid project, fix it cleanly and in isolation, give it back. The pronunciation +layer is the same move at larger scale — it makes Kokoro better for everyone and +makes our product possible at the same time. + +## The shape of the contribution (concrete) + +A **Prosody-Annotated Phoneme IR** as the contract between stages: + +``` +Token { text, phonemes|None, stress:-2..+2, emphasis:bool, + break_after_ms:int, source: {g2p_default|lexicon|llm_authored|human_override} } +``` + +Pipeline (each stage swappable, upstreamable pieces marked *): +1. Structure/markdown normalizer ← our md work, already built & hardened +2. Number/acronym/unit expander* (rule-based; where big TTS fails) +3. Lexicon resolver* (human > domain > rules > LLM > G2P default) +4. LLM IPA/prosody annotator + validation gate* (naive LLM→IPA is 31.6% PER; + validated against Kokoro's phoneme inventory → ~5.8%) +5. Kokoro adapter (emit `[word](/ipa/)` + `[word](+N)`, chunk <40k, `speed` callable) + +Graded by an **ASR-based eval harness** (WER against intended script) plus a +pronunciation unit-test corpus — build this first; it's what lets us *prove* +"correct," not just claim it. + +## Hard-won constraints (don't relearn these) + +- Kokoro stress markup `[word](+N)` works but is **subtle**; emotion markup + **never** works (not in its training data) — don't build emotion on Kokoro. +- **Validate every LLM-authored IPA** against the phoneme inventory before + injection; the failure mode is silent symbol inconsistency, not obvious garbage. +- Punctuation / respelling are the *reliable* prosody levers; SSML-style tags are + widely ignored by modern open TTS. +- Phoneme injection breaks ~40k chars → chunk. Word-level timestamps are free + forced alignment. `speed` takes a callable (unused pacing lever). diff --git a/README.md b/README.md index 0f1b9856..fd6b6ca8 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,18 @@ > **Fieldnote-Echo fork** of [remsky/Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) (currently based on `v0.6.0`) > -> **What this fork adds:** +> **Goal: correct, open, tunable narration.** Kokoro can *say* words but can't yet +> *guarantee* it says them correctly, and no open TTS gives authors real control over +> pronunciation. This fork is building that layer — correct-by-default pronunciation +> and prosody, tunable when it isn't — aimed at technical/e-learning narration. Not a +> naturalness contest, not singing. See [`ROADMAP.md`](./ROADMAP.md) and [`NORTH_STAR.md`](./NORTH_STAR.md). +> +> **Shipping today:** > - **Markdown-aware TTS normalization** — strips `**bold**`, `[links](url)`, headings, code blocks, tables, etc. before synthesis so LLM output sounds natural (toggle: `markdown_normalization` in `NormalizationOptions`, default on) -> - **Supply-chain hardened CI** — CUDA base images pinned by digest (in `docker-bake.hcl` and the release workflow), all GitHub Actions pinned to SHA, Sigstore build provenance, Trivy container scan, CycloneDX SBOM on every release +> - **Supply-chain hardened CI** — CUDA base images pinned by digest (`docker-bake.hcl` + release workflow), all GitHub Actions SHA-pinned, Sigstore build provenance, Trivy scan, CycloneDX SBOM per release +> - **Normalizer ReDoS hardening** — bounded the URL/email/markdown patterns that let a single request pin the event loop (also upstreamed: [remsky#489](https://github.com/remsky/Kokoro-FastAPI/pull/489)) > - **GHCR images** — `ghcr.io/fieldnote-echo/kokoro-fastapi-gpu` (GPU, amd64) > -> Everything else is upstream. We rebase on `remsky/master` periodically. +> Generally-useful fixes are contributed back upstream; we rebase on `remsky/master` periodically.

Kokoro TTS Banner diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000..8c16c2f9 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,56 @@ +# Fork Roadmap + +*Why this fork exists, and where it's going. Full rationale in [`NORTH_STAR.md`](./NORTH_STAR.md); market/technical research in [`NARRATION_STRATEGY.md`](./NARRATION_STRATEGY.md).* + +## North star + +**Correct, open, tunable narration.** Kokoro can *say* words; it can't yet +*guarantee* it says them correctly, and no open TTS gives authors real control +over pronunciation. This fork's purpose is to close that gap — pronunciation and +prosody an author (or the LLM drafting the script) can guarantee, correct by +default and tunable when it isn't — and to contribute the generally-useful parts +back upstream. + +**Not** a naturalness contest (large TTS wins that; fine) and **not** singing +(a physics-capped demo, permanently off the critical path). The wedge is +**correctness + control + cost.** + +## Good-faith upstreaming + +Parts that help every Kokoro user go to `remsky/Kokoro-FastAPI`; the product +(LMS integration, domain lexicons, the generation→narration loop) stays here. +The ReDoS fix ([remsky#489](https://github.com/remsky/Kokoro-FastAPI/pull/489)) +is the template: find a real gap, fix it cleanly in isolation, give it back. + +## Status + +| # | Milestone | State | +|---|---|---| +| 0 | Upstream sync to v0.6.0 | ✅ done | +| 0 | Supply-chain hardening (digest pins, SHA-pinned actions, provenance, SBOM, Trivy) | ✅ done | +| 0 | Normalizer ReDoS hardening | ✅ done · upstream PR #489 | +| 1 | Markdown-aware normalization (LLM output → speakable) | ✅ done | +| 2 | **ASR-based eval harness + pronunciation test corpus** | ⏭️ next | +| 3 | Deterministic number/acronym/unit expander | planned | +| 4 | Versioned pronunciation **lexicon** + resolver | planned | +| 5 | Prosody-Annotated Phoneme **IR** (the token contract) | planned | +| 6 | LLM IPA/prosody annotator **behind a validation gate** | planned | +| 7 | Blind A/B vs raw Kokoro + one commercial baseline | planned | +| H | *(horizon)* speech-to-singing WORLD retune — demo only | spike done | + +## Milestone 2 first (why) + +You can't claim "correct" or "less robotic" without measuring it. The harness +(WER via ASR against the intended script + a jargon/acronym unit-test corpus) +grades everything downstream and instantly catches pronunciation regressions. +It's the smallest, highest-leverage piece and it unblocks every later A/B claim. + +## Constraints banked (don't relearn) + +- Kokoro stress markup `[word](+N)` works but is **subtle**; emotion markup + **never** works (not in its training data). +- **Validate LLM-authored IPA** against Kokoro's phoneme inventory before + injection — naive LLM→IPA is ~31.6% phoneme-error; validated ~5.8%. +- Punctuation/respelling are the reliable prosody levers; SSML-style tags are + widely ignored. Phoneme injection breaks ~40k chars → chunk. Word timestamps + are free forced alignment. `speed` takes a callable (unused pacing lever). From 043098aa2d92d73de0a8ebc9c391fa152ca87145 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 19:44:15 -0500 Subject: [PATCH 17/17] =?UTF-8?q?fix(md):=20address=20bot=20review=20?= =?UTF-8?q?=E2=80=94=20URL=20regex,=20borderless=20tables,=20test=20flakin?= =?UTF-8?q?ess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bot findings on the fork PRs (Gemini/Qodo): - URL_PATTERN prefix (https?://|www\.|)+ has a '+' over a group with an empty alternative — a ReDoS smell and non-deterministic. Rewrote as two optionals (?:https?://)?(?:www\.)? — deterministic, same matches. handle_url uses only group(0), so the capturing->non-capturing shift is safe. - Table de-piping missed rows beyond the first in borderless multi-row tables and skipped 2-column borderless tables (1 pipe, failed the count>=2 gate). Replaced the anchored/adjacent heuristic with propagation of table status outward from each separator row; lone prose pipes stay intact. - Perf flood tests asserted <2.0s wall-clock; raised to <8.0s so they still catch the 12-65s O(n^2) blowups but don't flake on loaded CI runners. Adds borderless-table regression tests. 78 md/normalizer tests pass. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 41 +++++++++++-------- api/tests/test_markdown_normalizer.py | 38 +++++++++++++++-- 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 4941d7dc..95504beb 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -170,7 +170,10 @@ # run (O(n^2) — a 100KB run of 'a' hung the normalizer). The domain run # is also bounded at 253 chars, the DNS name length limit. r"(? str: text = _MD_BLOCKQUOTE.sub("", text) # Horizontal rules — remove entire line text = _MD_HORIZONTAL_RULE.sub("", text) - # Tables — remove separator rows, replace pipes with spaces on table - # rows. A row must have 2+ pipes AND either be pipe-anchored (leading or - # trailing |) or sit next to a separator row; prose pipes like - # "either a | b | c works" stay intact. - # Escaped pipes (\|) are literal content — hide them from the table - # logic, then restore as bare pipes at the end. + # Tables — remove separator rows and de-pipe the table body. A real table + # is a separator row (|---|---|) plus the contiguous pipe-bearing lines + # above and below it; propagate table status outward from each separator + # row. This catches borderless multi-row and 2-column tables (which the old + # "2+ pipes AND anchored/adjacent" heuristic missed) while leaving a lone + # prose line with pipes — "either a | b | c works", no adjacent separator — + # untouched. Escaped pipes (\|) are hidden as a sentinel first so they are + # neither counted as table syntax nor de-piped, then restored at the end. text = text.replace("\\|", "\x00EP\x00") lines = text.split("\n") is_sep_row = [_is_table_sep_row(line) for line in lines] + is_table_row = [False] * len(lines) + for i, sep in enumerate(is_sep_row): + if not sep: + continue + j = i - 1 + while j >= 0 and "|" in lines[j]: + is_table_row[j] = True + j -= 1 + j = i + 1 + while j < len(lines) and "|" in lines[j]: + is_table_row[j] = True + j += 1 for i, line in enumerate(lines): if is_sep_row[i]: lines[i] = "" - continue - if line.count("|") < 2: - continue - stripped = line.strip() - pipe_anchored = stripped.startswith("|") or stripped.endswith("|") - near_sep_row = (i > 0 and is_sep_row[i - 1]) or ( - i + 1 < len(lines) and is_sep_row[i + 1] - ) - if pipe_anchored or near_sep_row: + elif is_table_row[i]: lines[i] = line.replace("|", " ") text = "\n".join(lines).replace("\x00EP\x00", "|") diff --git a/api/tests/test_markdown_normalizer.py b/api/tests/test_markdown_normalizer.py index 05f43fac..d4e5b291 100644 --- a/api/tests/test_markdown_normalizer.py +++ b/api/tests/test_markdown_normalizer.py @@ -449,7 +449,7 @@ def test_unclosed_emphasis_flood_is_fast(): payload = marker * (100_000 // len(marker)) start = time.monotonic() handle_markdown(payload) - assert time.monotonic() - start < 2.0 + assert time.monotonic() - start < 8.0 # --------------------------------------------------------------------------- @@ -490,7 +490,7 @@ def test_adversarial_floods_are_fast(): for payload in payloads: start = time.monotonic() handle_markdown(payload) - assert time.monotonic() - start < 2.0, f"slow on {payload[:20]!r}..." + assert time.monotonic() - start < 8.0, f"slow on {payload[:20]!r}..." def test_emphasis_flood_scales_linearly(): @@ -517,7 +517,7 @@ def test_newline_and_whitespace_floods_are_fast(): for payload in ["\n" * 100_000, "\r\n" * 50_000, " \n" * 50_000]: start = time.monotonic() handle_markdown(payload) - assert time.monotonic() - start < 2.0 + assert time.monotonic() - start < 8.0 def test_long_wordrun_through_normalize_text_is_fast(): @@ -533,4 +533,34 @@ def test_long_wordrun_through_normalize_text_is_fast(): for payload in ["a" * 100_000, "a." * 50_000, "a@" * 50_000]: start = time.monotonic() normalize_text(payload, NormalizationOptions()) - assert time.monotonic() - start < 2.0 + assert time.monotonic() - start < 8.0 + + +# --------------------------------------------------------------------------- +# Borderless tables (Gemini finding: rows beyond the first, and 2-col tables, +# were left with raw pipes by the old anchored/adjacent heuristic) +# --------------------------------------------------------------------------- + + +def test_borderless_multirow_table_all_rows_depiped(): + md = "Name | Age | City\n--- | --- | ---\nAlice | 30 | NYC\nBob | 25 | LA" + result = handle_markdown(md) + assert "|" not in result + assert "---" not in result + for token in ("Alice", "Bob", "NYC", "LA"): + assert token in result + + +def test_borderless_two_column_table_depiped(): + """2-column borderless rows have only one pipe — the old count>=2 gate + skipped them entirely.""" + md = "Key | Value\n--- | ---\nfoo | bar" + result = handle_markdown(md) + assert "|" not in result + assert "foo" in result and "bar" in result + + +def test_lone_prose_pipes_still_intact(): + """A pipe-bearing prose line with no adjacent separator row is not a table.""" + text = "either a | b | c works" + assert handle_markdown(text) == text