diff --git a/api/src/services/text_processing/normalizer.py b/api/src/services/text_processing/normalizer.py index 13d76763..d84a10ad 100644 --- a/api/src/services/text_processing/normalizer.py +++ b/api/src/services/text_processing/normalizer.py @@ -155,11 +155,16 @@ MONEY_UNITS = {"$": ("dollar", "cent"), "£": ("pound", "pence"), "€": ("euro", "cent")} # Pre-compiled regex patterns for performance +# Lengths bounded to RFC limits (64/253) to prevent O(n^2) backtracking on floods. EMAIL_PATTERN = re.compile( - r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-z]{2,}\b", re.IGNORECASE + r"\b[a-zA-Z0-9._%+-]{1,64}@[a-zA-Z0-9.-]{1,253}\.[a-z]{2,}\b", re.IGNORECASE ) URL_PATTERN = re.compile( - r"(https?://|www\.|)+(localhost|[a-zA-Z0-9.-]+(\.(?:" + # Token-start lookbehind + 253-char domain bound: without them the domain + # branch rescans from every position of a long word run (O(n^2)). + r"(? st text = re.sub(r"(?<=[BCDFGHJ-NP-TV-Z])'?s\b", "'S", text) text = re.sub(r"(?<=X')S\b", "s", text) text = re.sub( - r"(?:[A-Za-z]\.){2,} [a-z]", lambda m: m.group().replace(".", "-"), text + # {2,12} not {2,} — real acronyms are short; unbounded backtracks O(n^2) on floods. + r"(?:[A-Za-z]\.){2,12} [a-z]", + lambda m: m.group().replace(".", "-"), + text, ) text = re.sub(r"(?i)(?<=[A-Z])\.(?=[A-Z])", "-", text) diff --git a/api/tests/test_normalizer_redos.py b/api/tests/test_normalizer_redos.py new file mode 100644 index 00000000..32cddae8 --- /dev/null +++ b/api/tests/test_normalizer_redos.py @@ -0,0 +1,55 @@ +"""Regression tests for ReDoS / catastrophic-backtracking in the text +normalizer. Each adversarial input previously drove normalize_text() into +super-linear time, blocking the async event loop (a single-request DoS, +since normalize_text runs synchronously inside the request path). +""" + +import time + +from api.src.services.text_processing.normalizer import normalize_text +from api.src.structures.schemas import NormalizationOptions + +OPTS = NormalizationOptions() +BUDGET_S = 2.0 # generous ceiling; pathological inputs previously took 10-65s + + +def _elapsed(payload: str) -> float: + start = time.monotonic() + normalize_text(payload, OPTS) + return time.monotonic() - start + + +def test_long_word_run_is_fast(): + """URL_PATTERN backtracked from every position of a dotless word run.""" + assert _elapsed("a" * 100_000) < BUDGET_S + + +def test_dotted_run_is_fast(): + """'a.a.a...' hit URL/EMAIL domain scans and the dotted-acronym handler.""" + assert _elapsed("a." * 50_000) < BUDGET_S + assert _elapsed(".a" * 50_000) < BUDGET_S + + +def test_at_sign_run_is_fast(): + """EMAIL_PATTERN local/domain runs on an '@' flood.""" + assert _elapsed("a@" * 50_000) < BUDGET_S + + +def test_scaling_is_linear(): + """4x input must cost far less than the ~16x a quadratic scan implies.""" + small = _elapsed("a." * 25_000) + large = _elapsed("a." * 100_000) + assert large / max(small, 1e-3) < 9.0 + + +def test_url_normalization_preserved(): + """The hardened patterns must still normalize real URLs and emails.""" + assert "google" in normalize_text("visit https://google.com now", OPTS) + assert "example" in normalize_text("see www.example.com/x", OPTS) + out = normalize_text("mail me at a.user@example.org please", OPTS) + assert "at" in out and "example" in out + + +def test_dotted_acronym_preserved(): + """The bounded acronym handler must still hyphenate 'U.S.A. b'.""" + assert "U-S-A-" in normalize_text("The U.S.A. beats all", OPTS)