From 7a5e0eac420ed4f012fc87d95fafb00f81049aeb Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 11:58:32 -0500 Subject: [PATCH 1/3] fix: prevent ReDoS in text normalizer (URL/email/acronym patterns) normalize_text() runs synchronously in the request path, so a single request of adversarial text could pin the event loop for tens of seconds via catastrophic regex backtracking: - URL_PATTERN: the domain branch '[a-zA-Z0-9.-]+' rescanned the whole tail from every position of a long word-char run (100KB of 'a' -> the normalizer hung). Added a token-start negative lookbehind and bounded the domain run to 253 chars (DNS limit). - EMAIL_PATTERN: unbounded local/domain runs scanned for '@' from every word boundary of an 'a.a.a...' flood. Bounded to RFC lengths (64/253). - Dotted-acronym handler '(?:[A-Za-z]\.){2,} [a-z]': unbounded '{2,}' backtracked O(n) per position on an 'a.a...' flood. Bounded to {2,12} (real acronyms are short); 'U.S.A. b' -> 'U-S-A- b' behavior preserved. Worst cases drop from 10-65s to <0.1s at 100KB. Adds api/tests/test_normalizer_redos.py covering each vector plus a linear-scaling guard; existing normalizer tests unchanged. Co-Authored-By: Claude Fable 5 --- .../services/text_processing/normalizer.py | 18 +++++- api/tests/test_normalizer_redos.py | 55 +++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 api/tests/test_normalizer_redos.py diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 13d76763..964e64e2 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -155,11 +155,19 @@ MONEY_UNITS = {"$": ("dollar", "cent"), "£": ("pound", "pence"), "€": ("euro", "cent")} # Pre-compiled regex patterns for performance +# Local part and domain are bounded at their RFC limits (64 / 253 chars): +# unbounded runs made every word boundary in an 'a.a.a...' flood scan the +# whole remaining string looking for '@' — O(n^2) catastrophic backtracking. EMAIL_PATTERN = re.compile( - r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-z]{2,}\b", re.IGNORECASE + r"\b[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-z]{2,}\b", re.IGNORECASE ) URL_PATTERN = re.compile( - r"(https?://|www\.|)+(localhost|[a-zA-Z0-9.-]+(\.(?:" + # Negative lookbehind: only attempt a match at token starts. Without it the + # domain branch re-scans from every position inside a long word-char run, + # which is O(n^2) (a 100KB run of 'a' hung the normalizer). The domain run + # is also bounded at 253 chars, the DNS name length limit. + r"(? st text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) text = re.sub(r"(?<=X')S\b", "s", text) text = re.sub( - r"(?:[A-Za-z]\.){2,} [a-z]", lambda m: m.group().replace(".", "-"), text + # Bound the acronym run (real dotted acronyms are short): an unbounded + # '{2,}' backtracks O(n) from every position of an 'a.a.a...' flood. + r"(?:[A-Za-z]\.){2,12} [a-z]", + lambda m: m.group().replace(".", "-"), + text, ) text = re.sub(r"(?i)(?<=[A-Z])\.(?=[A-Z])", "-", text) diff --git a/api/tests/test_normalizer_redos.py b/api/tests/test_normalizer_redos.py new file mode 100644 index 00000000..32cddae8 --- /dev/null +++ b/api/tests/test_normalizer_redos.py @@ -0,0 +1,55 @@ +"""Regression tests for ReDoS / catastrophic-backtracking in the text +normalizer. Each adversarial input previously drove normalize_text() into +super-linear time, blocking the async event loop (a single-request DoS, +since normalize_text runs synchronously inside the request path). +""" + +import time + +from api.src.services.text_processing.normalizer import normalize_text +from api.src.structures.schemas import NormalizationOptions + +OPTS = NormalizationOptions() +BUDGET_S = 2.0 # generous ceiling; pathological inputs previously took 10-65s + + +def _elapsed(payload: str) -> float: + start = time.monotonic() + normalize_text(payload, OPTS) + return time.monotonic() - start + + +def test_long_word_run_is_fast(): + """URL_PATTERN backtracked from every position of a dotless word run.""" + assert _elapsed("a" * 100_000) < BUDGET_S + + +def test_dotted_run_is_fast(): + """'a.a.a...' hit URL/EMAIL domain scans and the dotted-acronym handler.""" + assert _elapsed("a." * 50_000) < BUDGET_S + assert _elapsed(".a" * 50_000) < BUDGET_S + + +def test_at_sign_run_is_fast(): + """EMAIL_PATTERN local/domain runs on an '@' flood.""" + assert _elapsed("a@" * 50_000) < BUDGET_S + + +def test_scaling_is_linear(): + """4x input must cost far less than the ~16x a quadratic scan implies.""" + small = _elapsed("a." * 25_000) + large = _elapsed("a." * 100_000) + assert large / max(small, 1e-3) < 9.0 + + +def test_url_normalization_preserved(): + """The hardened patterns must still normalize real URLs and emails.""" + assert "google" in normalize_text("visit https://google.com now", OPTS) + assert "example" in normalize_text("see www.example.com/x", OPTS) + out = normalize_text("mail me at a.user@example.org please", OPTS) + assert "at" in out and "example" in out + + +def test_dotted_acronym_preserved(): + """The bounded acronym handler must still hyphenate 'U.S.A. b'.""" + assert "U-S-A-" in normalize_text("The U.S.A. beats all", OPTS) From dd091d0265fdcf01b256946714156b3891815bdc Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sat, 18 Jul 2026 19:51:30 -0500 Subject: [PATCH 2/3] fix: simplify URL_PATTERN prefix to avoid empty-alternative '+' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer flagged (https?://|www\.|)+ — a '+' quantifier over a group with an empty alternative, a ReDoS smell and needlessly non-deterministic. Rewrite as two optionals (?:https?://)?(?:www\.)? — deterministic, same accepted URLs. handle_url uses only group(0), so dropping the capturing group is safe. Co-Authored-By: Claude Fable 5 --- api/src/services/text_processing/normalizer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 964e64e2..4abcb8bd 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -167,7 +167,10 @@ # which is O(n^2) (a 100KB run of 'a' hung the normalizer). The domain run # is also bounded at 253 chars, the DNS name length limit. r"(? Date: Sun, 19 Jul 2026 09:27:48 -0500 Subject: [PATCH 3/3] docs: trim exposition comments per maintainer review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Condense the ReDoS-fix comments to one terse line each — keep the load-bearing 'why bounded' so the limits aren't removed later, drop the exposition. Co-Authored-By: Claude Fable 5 --- api/src/services/text_processing/normalizer.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 4abcb8bd..d84a10ad 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -155,21 +155,15 @@ MONEY_UNITS = {"$": ("dollar", "cent"), "£": ("pound", "pence"), "€": ("euro", "cent")} # Pre-compiled regex patterns for performance -# Local part and domain are bounded at their RFC limits (64 / 253 chars): -# unbounded runs made every word boundary in an 'a.a.a...' flood scan the -# whole remaining string looking for '@' — O(n^2) catastrophic backtracking. +# Lengths bounded to RFC limits (64/253) to prevent O(n^2) backtracking on floods. EMAIL_PATTERN = re.compile( r"\b[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-z]{2,}\b", re.IGNORECASE ) URL_PATTERN = re.compile( - # Negative lookbehind: only attempt a match at token starts. Without it the - # domain branch re-scans from every position inside a long word-char run, - # which is O(n^2) (a 100KB run of 'a' hung the normalizer). The domain run - # is also bounded at 253 chars, the DNS name length limit. + # Token-start lookbehind + 253-char domain bound: without them the domain + # branch rescans from every position of a long word run (O(n^2)). r"(? st text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) text = re.sub(r"(?<=X')S\b", "s", text) text = re.sub( - # Bound the acronym run (real dotted acronyms are short): an unbounded - # '{2,}' backtracks O(n) from every position of an 'a.a.a...' flood. + # {2,12} not {2,} — real acronyms are short; unbounded backtracks O(n^2) on floods. r"(?:[A-Za-z]\.){2,12} [a-z]", lambda m: m.group().replace(".", "-"), text,