From 860ca693e99e9ad8003958fe08f94dc7ef686122 Mon Sep 17 00:00:00 2001 From: Manuel Saelices Date: Thu, 2 Oct 2025 18:23:12 +0200 Subject: [PATCH 1/5] New `--all-reruns-need-to-pass` argument This addresses your requirement to verify that non-deterministic tests (that work ~90% of the time) pass consistently by requiring all reruns to pass after an initial failure. Signed-off-by: Manuel Saelices --- src/pytest_rerunfailures.py | 111 +++++++++++++-- tests/test_pytest_rerunfailures.py | 210 +++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+), 14 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index d17a1a7..0fd46d4 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -91,10 +91,18 @@ def pytest_addoption(parser): dest="fail_on_flaky", help="Fail the test run with exit code 7 if a flaky test passes on a rerun.", ) + group._addoption( + "--all-reruns-need-to-pass", + action="store_true", + dest="all_reruns_need_to_pass", + default=False, + help="If enabled, after an initial failure, all reruns must pass for the test to succeed.", + ) arg_type = "string" parser.addini("reruns", RERUNS_DESC, type=arg_type) parser.addini("reruns_delay", RERUNS_DELAY_DESC, type=arg_type) + parser.addini("all_reruns_need_to_pass", "If enabled, all reruns must pass after initial failure", type=arg_type) # making sure the options make sense @@ -162,6 +170,33 @@ def get_reruns_delay(item): return delay +def get_all_reruns_need_to_pass(item): + """Get whether all reruns need to pass from marker, config, or ini.""" + rerun_marker = _get_marker(item) + + # Check marker kwargs first + if rerun_marker is not None and "all_reruns_need_to_pass" in rerun_marker.kwargs: + return rerun_marker.kwargs["all_reruns_need_to_pass"] + + # Check command-line option + all_need_pass = item.session.config.getvalue("all_reruns_need_to_pass") + if all_need_pass is not None: + return all_need_pass + + # Check ini value + try: + all_need_pass = item.session.config.getini("all_reruns_need_to_pass") + if all_need_pass: + # Parse string values like "True", "true", "1", etc. + if isinstance(all_need_pass, str): + return all_need_pass.lower() in ("true", "1", "yes", "on") + return bool(all_need_pass) + except (TypeError, ValueError): + pass + + return False + + def get_reruns_condition(item): rerun_marker = _get_marker(item) @@ -324,9 +359,9 @@ def pytest_configure(config): # add flaky marker config.addinivalue_line( "markers", - "flaky(reruns=1, reruns_delay=0): mark test to re-run up " + "flaky(reruns=1, reruns_delay=0, all_reruns_need_to_pass=False): mark test to re-run up " "to 'reruns' times. Add a delay of 'reruns_delay' seconds " - "between re-runs.", + "between re-runs. If 'all_reruns_need_to_pass' is True, all reruns must pass.", ) if config.pluginmanager.hasplugin("xdist") and HAS_PYTEST_HANDLECRASHITEM: @@ -550,6 +585,7 @@ def pytest_runtest_protocol(item, nextitem): # first item if necessary check_options(item.session.config) delay = get_reruns_delay(item) + all_reruns_need_to_pass = get_all_reruns_need_to_pass(item) parallel = not is_master(item.config) db = item.session.config.failures_db item.execution_count = db.get_test_failures(item.nodeid) @@ -558,6 +594,10 @@ def pytest_runtest_protocol(item, nextitem): if item.execution_count > reruns: return True + # Track rerun results when all reruns need to pass + initial_failure_occurred = False + rerun_results = [] # Track result of each rerun (True=passed, False=failed) + need_to_run = True while need_to_run: item.execution_count += 1 @@ -566,23 +606,66 @@ def pytest_runtest_protocol(item, nextitem): for report in reports: # 3 reports: setup, call, teardown report.rerun = item.execution_count - 1 - if _should_not_rerun(item, report, reruns): - # last run or no failure detected, log normally - item.ihook.pytest_runtest_logreport(report=report) + + # Track initial failure for all_reruns_need_to_pass mode + if all_reruns_need_to_pass and report.when == "call" and report.failed and item.execution_count == 1: + initial_failure_occurred = True + + # Track rerun results (after initial failure) - only track call phase + if all_reruns_need_to_pass and initial_failure_occurred and item.execution_count > 1 and report.when == "call": + rerun_results.append(not report.failed) # True if passed, False if failed + + # In all_reruns_need_to_pass mode with initial failure, override normal behavior + if all_reruns_need_to_pass and initial_failure_occurred: + # execution_count starts at 1, so: + # - execution_count==1: initial run (failed) + # - execution_count==2..reruns+1: reruns (must run all of them) + is_last_rerun = item.execution_count > reruns + + if is_last_rerun: + # Last run, check if all reruns passed + if any(not r for r in rerun_results): + # At least one rerun failed, mark final outcome as failed + if report.when == "call": + report.outcome = "failed" + # Log the final report + item.ihook.pytest_runtest_logreport(report=report) + else: + # Not the last rerun yet + # Only trigger rerun after processing the call phase + if report.when == "call": + report.outcome = "rerun" + time.sleep(delay) + if not parallel or works_with_current_xdist(): + item.ihook.pytest_runtest_logreport(report=report) + + _remove_cached_results_from_failed_fixtures(item) + _remove_failed_setup_state_from_session(item) + break # trigger rerun + else: + # For setup/teardown, just log normally + item.ihook.pytest_runtest_logreport(report=report) else: - # failure detected and reruns not exhausted, since i < reruns - report.outcome = "rerun" - time.sleep(delay) + # Normal rerun behavior + should_not_rerun = _should_not_rerun(item, report, reruns) - if not parallel or works_with_current_xdist(): - # will rerun test, log intermediate result + if should_not_rerun: + # last run or no failure detected, log normally item.ihook.pytest_runtest_logreport(report=report) + else: + # failure detected and reruns not exhausted + report.outcome = "rerun" + time.sleep(delay) + + if not parallel or works_with_current_xdist(): + # will rerun test, log intermediate result + item.ihook.pytest_runtest_logreport(report=report) - # cleanin item's cashed results from any level of setups - _remove_cached_results_from_failed_fixtures(item) - _remove_failed_setup_state_from_session(item) + # cleanin item's cashed results from any level of setups + _remove_cached_results_from_failed_fixtures(item) + _remove_failed_setup_state_from_session(item) - break # trigger rerun + break # trigger rerun else: need_to_run = False diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index afb706f..4122fdf 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1357,3 +1357,213 @@ def test_1(session_fixture, function_fixture): result = testdir.runpytest() assert_outcomes(result, passed=0, failed=1, rerun=1) result.stdout.fnmatch_lines("session teardown") + + +def test_all_reruns_need_to_pass_disabled_by_default(testdir): + """Test that default behavior is unchanged (flag disabled by default).""" + testdir.makepyfile( + """ + import py + def test_default_behavior(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on second attempt + """ + ) + result = testdir.runpytest("--reruns", "3") + # Should pass because one successful rerun is enough (default behavior) + assert_outcomes(result, passed=1, rerun=1) + + +def test_all_reruns_need_to_pass_all_pass(testdir): + """Test that when all reruns pass, test passes.""" + testdir.makepyfile( + """ + import py + def test_all_pass(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass because all 3 reruns pass + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_some_fail(testdir): + """Test that when some reruns fail, test fails.""" + testdir.makepyfile( + """ + import py + def test_some_fail(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + # Fail on attempt 0 (initial) and attempt 2 (second rerun) + if count == 0 or count == 2: + raise Exception(f'Fail on attempt {count}') + # Pass on attempts 1 and 3 + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should fail because rerun 2 fails + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_all_reruns_need_to_pass_all_fail(testdir): + """Test that when all reruns fail, test fails.""" + testdir.makepyfile( + """ + def test_all_fail(): + raise Exception('Always fail') + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should fail because all reruns fail + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_all_reruns_need_to_pass_marker_override(testdir): + """Test that marker can override command-line flag.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.mark.flaky(reruns=3, all_reruns_need_to_pass=True) + def test_marker_override(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + # Not passing --all-reruns-need-to-pass on command line, but marker enables it + result = testdir.runpytest("--verbose") + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_marker_can_disable(testdir): + """Test that marker can disable flag even when set on command line.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.mark.flaky(reruns=3, all_reruns_need_to_pass=False) + def test_marker_disables(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on second attempt + """ + ) + # Passing --all-reruns-need-to-pass, but marker disables it + result = testdir.runpytest("--all-reruns-need-to-pass", "--verbose") + # Should pass with just one successful rerun (default behavior) + assert_outcomes(result, passed=1, rerun=1) + + +def test_all_reruns_need_to_pass_with_only_rerun(testdir): + """Test interaction with --only-rerun flag.""" + testdir.makepyfile( + """ + import py + def test_with_only_rerun(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise ValueError('Fail on first attempt') + # Pass on subsequent attempts + """ + ) + result = testdir.runpytest( + "--reruns", "3", + "--all-reruns-need-to-pass", + "--only-rerun", "ValueError" + ) + # Should pass because all reruns pass + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_initial_pass(testdir): + """Test that flag has no effect if test passes on first attempt.""" + testdir.makepyfile( + """ + def test_initial_pass(): + pass # Always passes + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass without any reruns + assert_outcomes(result, passed=1, rerun=0) + + +def test_all_reruns_need_to_pass_zero_reruns(testdir): + """Test that flag has no effect with reruns=0.""" + testdir.makepyfile( + """ + def test_zero_reruns(): + raise Exception('Fail') + """ + ) + result = testdir.runpytest("--reruns", "0", "--all-reruns-need-to-pass") + # Should fail without any reruns + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_all_reruns_need_to_pass_setup_failure(testdir): + """Test behavior when setup fails - only call phase is tracked.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.fixture + def failing_fixture(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count < 2: + raise Exception(f'Fixture fails on attempt {count}') + return "ok" + + def test_setup_failure(failing_fixture): + pass + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Note: all_reruns_need_to_pass only tracks call phase failures, not setup/teardown + # Setup eventually passes, so test passes + assert_outcomes(result, passed=1, rerun=2) + + +def test_all_reruns_need_to_pass_command_line(testdir): + """Test that command line flag works as expected.""" + testdir.makepyfile( + """ + import py + def test_cli_flag(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass because all reruns pass + assert_outcomes(result, passed=1, rerun=3) From b59ed10fd80648b7ac14d185d7dd1d98d7ca3089 Mon Sep 17 00:00:00 2001 From: Manuel Saelices Date: Fri, 22 May 2026 12:04:11 +0200 Subject: [PATCH 2/5] Format with ruff and wrap long lines --- src/pytest_rerunfailures.py | 41 ++++++++++++++++++++++-------- tests/test_pytest_rerunfailures.py | 4 +-- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 0fd46d4..a6c2308 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -96,13 +96,20 @@ def pytest_addoption(parser): action="store_true", dest="all_reruns_need_to_pass", default=False, - help="If enabled, after an initial failure, all reruns must pass for the test to succeed.", + help=( + "If enabled, after an initial failure, all reruns must pass " + "for the test to succeed." + ), ) arg_type = "string" parser.addini("reruns", RERUNS_DESC, type=arg_type) parser.addini("reruns_delay", RERUNS_DELAY_DESC, type=arg_type) - parser.addini("all_reruns_need_to_pass", "If enabled, all reruns must pass after initial failure", type=arg_type) + parser.addini( + "all_reruns_need_to_pass", + "If enabled, all reruns must pass after initial failure", + type=arg_type, + ) # making sure the options make sense @@ -359,9 +366,10 @@ def pytest_configure(config): # add flaky marker config.addinivalue_line( "markers", - "flaky(reruns=1, reruns_delay=0, all_reruns_need_to_pass=False): mark test to re-run up " - "to 'reruns' times. Add a delay of 'reruns_delay' seconds " - "between re-runs. If 'all_reruns_need_to_pass' is True, all reruns must pass.", + "flaky(reruns=1, reruns_delay=0, all_reruns_need_to_pass=False): " + "mark test to re-run up to 'reruns' times. Add a delay of " + "'reruns_delay' seconds between re-runs. If " + "'all_reruns_need_to_pass' is True, all reruns must pass.", ) if config.pluginmanager.hasplugin("xdist") and HAS_PYTEST_HANDLECRASHITEM: @@ -608,14 +616,27 @@ def pytest_runtest_protocol(item, nextitem): report.rerun = item.execution_count - 1 # Track initial failure for all_reruns_need_to_pass mode - if all_reruns_need_to_pass and report.when == "call" and report.failed and item.execution_count == 1: + if ( + all_reruns_need_to_pass + and report.when == "call" + and report.failed + and item.execution_count == 1 + ): initial_failure_occurred = True # Track rerun results (after initial failure) - only track call phase - if all_reruns_need_to_pass and initial_failure_occurred and item.execution_count > 1 and report.when == "call": - rerun_results.append(not report.failed) # True if passed, False if failed - - # In all_reruns_need_to_pass mode with initial failure, override normal behavior + if ( + all_reruns_need_to_pass + and initial_failure_occurred + and item.execution_count > 1 + and report.when == "call" + ): + rerun_results.append( + not report.failed + ) # True if passed, False if failed + + # In all_reruns_need_to_pass mode with initial failure, + # override normal behavior if all_reruns_need_to_pass and initial_failure_occurred: # execution_count starts at 1, so: # - execution_count==1: initial run (failed) diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 4122fdf..6db0dd1 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1490,9 +1490,7 @@ def test_with_only_rerun(): """ ) result = testdir.runpytest( - "--reruns", "3", - "--all-reruns-need-to-pass", - "--only-rerun", "ValueError" + "--reruns", "3", "--all-reruns-need-to-pass", "--only-rerun", "ValueError" ) # Should pass because all reruns pass assert_outcomes(result, passed=1, rerun=3) From faf5f47e24a2f64b63c487e77e2243db22653c2b Mon Sep 17 00:00:00 2001 From: Giovanni Date: Wed, 8 Jul 2026 15:38:34 +0100 Subject: [PATCH 3/5] Add pytest_rerunfailures_rerun_policy hook for per-failure rerun policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plugin can now return a RerunPolicy for a test's first failure to give that specific failure class its own rerun count / all-reruns-need-to-pass mode and an outcome tag, without changing how other failures are rerun. Motivation: distinguish transient provider-infra failures (5xx / 429 / timeouts) from behavioral failures in an eval suite — give infra failures a single rerun with pass-on-recovery semantics, and tag the ones that persist so a sustained outage can be reported as "infra-degraded" instead of a wall of red — while behavioral failures keep the strict --reruns N --all-reruns-need-to-pass net. This per-failure-type policy can't be expressed via --only-rerun (it is bypassed under --all-reruns-need-to-pass) or a global --reruns count. - New hookspec pytest_rerunfailures_rerun_policy(item, report, call) in a dedicated pytest_rerunfailures_newhooks module (so add_hookspecs sees only the spec, not the plugin's own hook impls). Returns a RerunPolicy or None (default). - RerunPolicy(reruns, all_reruns_need_to_pass, tag): fields left None keep defaults; tag is stamped on the final failed report as report.rerun_tag (survives xdist). - Called once per test on its first failing report; the protocol applies the override after the first attempt and tags the final failure. Default behavior is unchanged. - Tests, CHANGES, README, and packaging (py-modules) for the new module. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGES.rst | 5 +- README.rst | 27 +++++ pyproject.toml | 4 + src/pytest_rerunfailures.py | 65 ++++++++++++ src/pytest_rerunfailures_newhooks.py | 27 +++++ tests/test_pytest_rerunfailures.py | 146 +++++++++++++++++++++++++++ 6 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 src/pytest_rerunfailures_newhooks.py diff --git a/CHANGES.rst b/CHANGES.rst index 7960238..0a312a9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,10 @@ Changelog 16.4 (unreleased) ----------------- -- Nothing changed yet. +- Added the ``pytest_rerunfailures_rerun_policy`` hook: plugins can return a + ``RerunPolicy`` to give a specific failure class its own rerun count / + all-reruns-need-to-pass mode and an outcome tag (``report.rerun_tag``), without + changing how other failures are rerun. 16.3 (2026-05-22) diff --git a/README.rst b/README.rst index 30d2669..a3c7193 100644 --- a/README.rst +++ b/README.rst @@ -236,6 +236,33 @@ Each retried attempt's traceback is appended to the ``rerun test summary info`` section. The section is emitted automatically when the flag is set, so ``-rR`` is not required. +Per-failure rerun policy (hook) +------------------------------- + +Plugins can give a *specific failure class* its own rerun count and semantics via the +``pytest_rerunfailures_rerun_policy`` hook, without changing how other failures are +rerun. The hook is called once per test, on its first failing report, and returns a +``RerunPolicy`` (or ``None`` for the default behavior): + +.. code-block:: python + + # conftest.py + from pytest_rerunfailures import RerunPolicy + + def pytest_rerunfailures_rerun_policy(item, report, call): + # e.g. give transient provider/infra errors one rerun, pass on recovery, + # and tag the failure if it persists — while other failures keep the + # global --reruns / --all-reruns-need-to-pass behavior. + exc = call.excinfo.value if call.excinfo else None + if isinstance(exc, ConnectionError): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return None + +``RerunPolicy`` fields left ``None`` keep the plugin's default for that item. If ``tag`` +is set and the failure persists after its reruns, it is stamped on the final failed +report as ``report.rerun_tag`` (an attribute that survives xdist serialization), so a +consumer can tell that failure class apart afterwards. + Output ------ diff --git a/pyproject.toml b/pyproject.toml index 2800983..f0fe8c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,10 @@ entry-points.pytest11.rerunfailures = "pytest_rerunfailures" [tool.setuptools.dynamic] readme = { file = [ "HEADER.rst", "README.rst", "CHANGES.rst" ] } +[tool.setuptools] +package-dir = { "" = "src" } +py-modules = [ "pytest_rerunfailures", "pytest_rerunfailures_newhooks" ] + [tool.ruff] fix = true lint.select = [ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 09beb82..0cf249d 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -10,6 +10,7 @@ import traceback import warnings from contextlib import suppress +from dataclasses import dataclass import pytest from _pytest.outcomes import fail @@ -46,6 +47,37 @@ def works_with_current_xdist(): RERUNS_DELAY_DESC = "add time (seconds) delay between reruns." +@dataclass(frozen=True) +class RerunPolicy: + """Per-failure rerun policy returned by ``pytest_rerunfailures_rerun_policy``. + + Any field left ``None`` keeps the plugin's default for that item. ``tag``, if set, + is stamped onto the final failed report as ``report.rerun_tag`` (an attribute that + survives xdist worker->controller serialization), so a consumer can tell this + failure class apart after the run. + + Attributes: + reruns: Rerun count for this failure (overrides ``--reruns`` / the marker). + all_reruns_need_to_pass: Override all-reruns-need-to-pass for this failure. + tag: Label stamped on the final failed report as ``report.rerun_tag``. + """ + + reruns: int | None = None + all_reruns_need_to_pass: bool | None = None + tag: str | None = None + + +def pytest_addhooks(pluginmanager): + """Register this plugin's own hookspecs (see ``pytest_rerunfailures_newhooks``). + + The specs live in a dedicated module so ``add_hookspecs`` sees only them, not this + module's own hook implementations. + """ + import pytest_rerunfailures_newhooks + + pluginmanager.add_hookspecs(pytest_rerunfailures_newhooks) + + # command line options def pytest_addoption(parser): group = parser.getgroup( @@ -409,6 +441,12 @@ def _should_not_rerun(item, report, reruns): ) +def _apply_rerun_tag(report, rerun_tag): + """Stamp a rerun-policy tag on a final failed report as ``report.rerun_tag``.""" + if rerun_tag and report.failed: + report.rerun_tag = rerun_tag + + def is_master(config): return not (hasattr(config, "workerinput") or hasattr(config, "slaveinput")) @@ -642,6 +680,13 @@ def pytest_runtest_makereport(item, call): item, result, call.excinfo ) + # On the first failure, ask plugins for a per-failure rerun policy so the protocol + # can give this failure class its own rerun count / semantics and outcome tag. + if result.failed and not hasattr(item, "_rerunfailures_policy"): + item._rerunfailures_policy = item.ihook.pytest_rerunfailures_rerun_policy( + item=item, report=result, call=call + ) + def pytest_runtest_protocol(item, nextitem): """ @@ -672,6 +717,8 @@ def pytest_runtest_protocol(item, nextitem): # Track rerun results when all reruns need to pass initial_failure_occurred = False rerun_results = [] # Track result of each rerun (True=passed, False=failed) + rerun_tag = None # From a per-failure rerun policy; stamped on the final failure. + policy_applied = False need_to_run = True while need_to_run: @@ -679,6 +726,21 @@ def pytest_runtest_protocol(item, nextitem): item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) reports = runtestprotocol(item, nextitem=nextitem, log=False) + # Apply a per-failure rerun policy (pytest_rerunfailures_rerun_policy) once the + # first attempt has run and makereport has classified its failure. Lets a plugin + # override the rerun count / all-reruns mode for THIS item and tag its final + # failure, without changing how other failures are rerun. + if not policy_applied: + policy = getattr(item, "_rerunfailures_policy", None) + if policy is not None: + policy_applied = True + if policy.reruns is not None: + reruns = policy.reruns + db.set_test_reruns(item.nodeid, reruns) + if policy.all_reruns_need_to_pass is not None: + all_reruns_need_to_pass = policy.all_reruns_need_to_pass + rerun_tag = policy.tag + for report in reports: # 3 reports: setup, call, teardown report.rerun = item.execution_count - 1 @@ -717,6 +779,7 @@ def pytest_runtest_protocol(item, nextitem): if report.when == "call": report.outcome = "failed" # Log the final report + _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # Not the last rerun yet @@ -732,6 +795,7 @@ def pytest_runtest_protocol(item, nextitem): break # trigger rerun else: # For setup/teardown, just log normally + _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # Normal rerun behavior @@ -739,6 +803,7 @@ def pytest_runtest_protocol(item, nextitem): if should_not_rerun: # last run or no failure detected, log normally + _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # failure detected and reruns not exhausted diff --git a/src/pytest_rerunfailures_newhooks.py b/src/pytest_rerunfailures_newhooks.py new file mode 100644 index 0000000..bea9200 --- /dev/null +++ b/src/pytest_rerunfailures_newhooks.py @@ -0,0 +1,27 @@ +"""Hook specifications added by the smith fork of pytest-rerunfailures. + +Kept in a dedicated module so ``PluginManager.add_hookspecs`` registers only these +specs and not the plugin's own ``pytest_*`` hook implementations (pytest treats any +``pytest_``-prefixed function in a namespace as a hookspec). +""" + +import pytest + + +@pytest.hookspec(firstresult=True) +def pytest_rerunfailures_rerun_policy(item, report, call): + """Return a ``RerunPolicy`` for a test's first failure, or ``None`` for default. + + Called once per test, on its first failing report (setup or call). Lets a plugin + give a specific failure class (e.g. provider-infra errors) its own rerun count / + semantics and an outcome tag, without affecting how other failures are rerun. The + first non-None result wins. + + Args: + item: The test item that failed. + report: The failing ``TestReport`` (``.when`` is ``"setup"`` or ``"call"``). + call: The ``CallInfo`` (``call.excinfo`` carries the exception). + + Returns: + A ``pytest_rerunfailures.RerunPolicy``, or ``None`` for default behavior. + """ diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 8fb2cf7..a0bd6da 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1734,3 +1734,149 @@ def test_pass(): result = testdir.runpytest("--reruns-mode", "bogus") assert result.ret != 0 + + +# --- pytest_rerunfailures_rerun_policy: per-failure rerun policy hook --- + + +def test_rerun_policy_none_keeps_global_behavior(testdir): + """A policy hook that returns None does not change the global rerun behavior.""" + testdir.makeconftest( + """ + def pytest_rerunfailures_rerun_policy(item, report, call): + return None + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + result = testdir.runpytest("--reruns", "2") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_rerun_policy_overrides_rerun_count(testdir): + """A policy can cap the rerun count for its failure below the global --reruns.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + # Global --reruns is 5, but the policy caps this failure at a single rerun. + result = testdir.runpytest("--reruns", "5") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_recovers_on_single_rerun(testdir): + """A tagged failure that recovers on its one rerun passes under all-reruns mode.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + """ + ) + testdir.makepyfile( + """ + import py + def test_transient(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('transient failure') + """ + ) + # Global mode is all-reruns-need-to-pass with 3 reruns; the policy overrides it + # to "1 rerun, pass on recovery" for this failure. + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_overrides_all_reruns_need_to_pass(testdir): + """A policy can disable all-reruns-need-to-pass for its failure (sustained fail).""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('always down')") + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # One rerun (not three), and it fails -> a single failure. + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_only_applies_to_matching_failures(testdir): + """The policy can target one failure class; others keep default behavior.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + if call.excinfo is not None and call.excinfo.typename == "InfraError": + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return None + """ + ) + testdir.makepyfile( + """ + class InfraError(Exception): + pass + + def test_infra(): + raise InfraError('provider down') + + def test_behavioral(): + raise AssertionError('real bug') + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # test_infra: policy -> 1 rerun; test_behavioral: default -> 3. Both fail. + assert_outcomes(result, passed=0, failed=2, rerun=4) + + +def test_rerun_policy_tags_final_failure(testdir): + """A policy tag is stamped on the final failed report as ``report.rerun_tag``.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, tag="infra") + def pytest_runtest_logreport(report): + if report.when == "call" and report.failed: + tag = getattr(report, "rerun_tag", "") + if tag: + import py + py.path.local(__file__).dirpath().join("tag.res").write(tag) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + result = testdir.runpytest("--reruns", "1") + assert_outcomes(result, passed=0, failed=1, rerun=1) + assert testdir.tmpdir.join("tag.res").read() == "infra" + + +@pytest.mark.skipif(not has_xdist, reason="requires pytest-xdist") +def test_rerun_policy_under_xdist(testdir): + """The policy override + recovery work under xdist (policy per-item).""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + """ + ) + testdir.makepyfile( + """ + import py + def test_transient(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('transient failure') + """ + ) + result = testdir.runpytest("-n", "1", "--reruns", "3", "--all-reruns-need-to-pass") + assert_outcomes(result, passed=1, rerun=1) From ed5e5af3c44821a314feb42076c4c4e5a3ae47ea Mon Sep 17 00:00:00 2001 From: Giovanni Date: Thu, 9 Jul 2026 16:20:44 +0100 Subject: [PATCH 4/5] Address adversarial review: docstring accuracy + policy-hook test coverage Docstring fix (factual): - pytest_rerunfailures_rerun_policy report.when may be "setup", "call", OR "teardown" (the makereport guard has no when-filter); the docstring wrongly said "setup or call". Also document that the policy is locked at the first failure and not recomputed on later attempts. New tests (coverage gaps the review found): - no policy hook implemented at all -> global behavior unchanged (additive contract) - a policy can RAISE the rerun count above the global --reruns - firstresult=True: the first non-None policy wins over a later implementation - all_reruns_need_to_pass=False override takes effect with reruns>=2 (the prior tests only used reruns=1, where the flag is a no-op; mutation-tested: neutralizing the override branch makes this test fail) - policy is locked at the first failure: a differently-classed failure on a rerun inherits the first policy and is not recomputed Full suite: 152 passed. Co-Authored-By: Claude Opus 4.8 --- src/pytest_rerunfailures_newhooks.py | 14 ++-- tests/test_pytest_rerunfailures.py | 118 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 5 deletions(-) diff --git a/src/pytest_rerunfailures_newhooks.py b/src/pytest_rerunfailures_newhooks.py index bea9200..a08fa9c 100644 --- a/src/pytest_rerunfailures_newhooks.py +++ b/src/pytest_rerunfailures_newhooks.py @@ -12,14 +12,18 @@ def pytest_rerunfailures_rerun_policy(item, report, call): """Return a ``RerunPolicy`` for a test's first failure, or ``None`` for default. - Called once per test, on its first failing report (setup or call). Lets a plugin - give a specific failure class (e.g. provider-infra errors) its own rerun count / - semantics and an outcome tag, without affecting how other failures are rerun. The - first non-None result wins. + Consulted once per test, on its first failing report (``.when`` may be ``"setup"``, + ``"call"``, or ``"teardown"``). The returned policy is locked to the item's WHOLE rerun + sequence -- it is not recomputed on later attempts, so a differently-classed failure on + a rerun inherits this policy (the rerun count / mode is fixed by the first failure). Lets + a plugin give a specific failure class (e.g. provider-infra errors) its own rerun count / + semantics and an outcome tag, without affecting how other failures are rerun. The first + non-None result wins. Args: item: The test item that failed. - report: The failing ``TestReport`` (``.when`` is ``"setup"`` or ``"call"``). + report: The first failing ``TestReport`` (``.when`` is ``"setup"``, ``"call"``, + or ``"teardown"``). call: The ``CallInfo`` (``call.excinfo`` carries the exception). Returns: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index a0bd6da..7075502 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1880,3 +1880,121 @@ def test_transient(): ) result = testdir.runpytest("-n", "1", "--reruns", "3", "--all-reruns-need-to-pass") assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_no_hook_impl_keeps_global_behavior(testdir): + """With NO policy hook implemented at all, global rerun behavior is unchanged. + + Guards the additive contract: suites that never implement the hook must behave + exactly as before (this is the common case). + """ + testdir.makepyfile("def test_fail(): raise Exception('boom')") + result = testdir.runpytest("--reruns", "2") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_rerun_policy_can_increase_rerun_count(testdir): + """A policy can RAISE the rerun count for its failure above the global --reruns.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=3) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + # Global --reruns is 1, but the policy raises this failure's count to 3. + result = testdir.runpytest("--reruns", "1") + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_rerun_policy_firstresult_first_non_none_wins(testdir): + """firstresult=True: the first non-None policy wins over a later implementation.""" + testdir.makepyfile( + plugin_first=""" + import pytest + from pytest_rerunfailures import RerunPolicy + @pytest.hookimpl(tryfirst=True) + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1) + """ + ) + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=5) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + testdir.syspathinsert() # make plugin_first importable for `-p` + # plugin_first is consulted first (tryfirst) and returns non-None, so its reruns=1 + # wins over the conftest's reruns=5 (which would give rerun=5 if it won). + result = testdir.runpytest("-p", "plugin_first", "--reruns", "3") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_disables_all_reruns_stops_on_first_pass(testdir): + """Overriding all_reruns_need_to_pass=False takes effect with reruns>=2. + + Distinguishes the two modes (which are indistinguishable at reruns=1): a + fail/pass/fail sequence stops on the first passing rerun under the override + (passed), whereas global all-reruns-need-to-pass would run every rerun and fail + because a later rerun failed. + """ + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=2, all_reruns_need_to_pass=False) + """ + ) + testdir.makepyfile( + """ + import py + def test_flaky(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count != 1: # fail (0), pass (1), fail (2) + raise Exception('flaky failure') + """ + ) + result = testdir.runpytest("--reruns", "2", "--all-reruns-need-to-pass") + # Override -> normal mode: stops on the first passing rerun. Without the override + # (all-reruns-need-to-pass) it would be passed=0, failed=1, rerun=2. + assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_locked_at_first_failure(testdir): + """The policy is fixed at the first failure; a differently-classed rerun failure + inherits it and is NOT recomputed.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + # Only the first failure (InfraError) ever reaches this hook; give it 1 rerun. + if call.excinfo is not None and call.excinfo.typename == "InfraError": + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + return RerunPolicy(reruns=3) # would apply if (wrongly) recomputed on the rerun + """ + ) + testdir.makepyfile( + """ + import py + class InfraError(Exception): + pass + + def test_mixed(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise InfraError('provider blip') # first failure -> policy reruns=1 + raise AssertionError('real bug on the rerun') # different class on the rerun + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Locked to the infra policy (1 rerun), NOT recomputed to reruns=3 for the behavioral + # failure that surfaces on the rerun. + assert_outcomes(result, passed=0, failed=1, rerun=1) From 843368996fea0c5282f55f2d9936dc46b460b359 Mon Sep 17 00:00:00 2001 From: Giovanni Date: Thu, 9 Jul 2026 16:39:04 +0100 Subject: [PATCH 5/5] Drop the unused RerunPolicy.tag / report.rerun_tag feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tag/rerun_tag labeling was the first-generation mechanism for a consumer to identify a failure class after the run. The smith eval harness — the only consumer — never adopted it: it classifies failures per-attempt in its own conftest (report.verdict_infra, infra only if every attempt was infra) rather than reading report.rerun_tag, precisely so a behavioral regression surfacing on a rerun is never masked. Nothing else uses tag, and it was under-tested (the all-reruns tag-stamping branch + xdist serialization were unverified), so remove it. Removed: RerunPolicy.tag, _apply_rerun_tag, the rerun_tag stamping in the protocol, the tag test, and the tag docs (README/CHANGES/hookspec). The per-failure `reruns` / `all_reruns_need_to_pass` overrides — the actual infra-rerun mechanism — are unchanged and fully tested. Full suite: 151 passed. Co-Authored-By: Claude Opus 4.8 --- CHANGES.rst | 3 +-- README.rst | 14 ++++++------- src/pytest_rerunfailures.py | 24 ++++----------------- src/pytest_rerunfailures_newhooks.py | 3 +-- tests/test_pytest_rerunfailures.py | 31 +++++----------------------- 5 files changed, 17 insertions(+), 58 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0a312a9..f186682 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,8 +6,7 @@ Changelog - Added the ``pytest_rerunfailures_rerun_policy`` hook: plugins can return a ``RerunPolicy`` to give a specific failure class its own rerun count / - all-reruns-need-to-pass mode and an outcome tag (``report.rerun_tag``), without - changing how other failures are rerun. + all-reruns-need-to-pass mode, without changing how other failures are rerun. 16.3 (2026-05-22) diff --git a/README.rst b/README.rst index a3c7193..3974f48 100644 --- a/README.rst +++ b/README.rst @@ -250,18 +250,16 @@ rerun. The hook is called once per test, on its first failing report, and return from pytest_rerunfailures import RerunPolicy def pytest_rerunfailures_rerun_policy(item, report, call): - # e.g. give transient provider/infra errors one rerun, pass on recovery, - # and tag the failure if it persists — while other failures keep the - # global --reruns / --all-reruns-need-to-pass behavior. + # e.g. give transient provider/infra errors one rerun, pass on recovery — + # while other failures keep the global --reruns / --all-reruns-need-to-pass + # behavior. exc = call.excinfo.value if call.excinfo else None if isinstance(exc, ConnectionError): - return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) return None -``RerunPolicy`` fields left ``None`` keep the plugin's default for that item. If ``tag`` -is set and the failure persists after its reruns, it is stamped on the final failed -report as ``report.rerun_tag`` (an attribute that survives xdist serialization), so a -consumer can tell that failure class apart afterwards. +``RerunPolicy`` fields left ``None`` keep the plugin's default for that item. The policy +is fixed at the item's first failure and applies to its whole rerun sequence. Output ------ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 0cf249d..3c5a8bc 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -51,20 +51,15 @@ def works_with_current_xdist(): class RerunPolicy: """Per-failure rerun policy returned by ``pytest_rerunfailures_rerun_policy``. - Any field left ``None`` keeps the plugin's default for that item. ``tag``, if set, - is stamped onto the final failed report as ``report.rerun_tag`` (an attribute that - survives xdist worker->controller serialization), so a consumer can tell this - failure class apart after the run. + Any field left ``None`` keeps the plugin's default for that item. Attributes: reruns: Rerun count for this failure (overrides ``--reruns`` / the marker). all_reruns_need_to_pass: Override all-reruns-need-to-pass for this failure. - tag: Label stamped on the final failed report as ``report.rerun_tag``. """ reruns: int | None = None all_reruns_need_to_pass: bool | None = None - tag: str | None = None def pytest_addhooks(pluginmanager): @@ -441,12 +436,6 @@ def _should_not_rerun(item, report, reruns): ) -def _apply_rerun_tag(report, rerun_tag): - """Stamp a rerun-policy tag on a final failed report as ``report.rerun_tag``.""" - if rerun_tag and report.failed: - report.rerun_tag = rerun_tag - - def is_master(config): return not (hasattr(config, "workerinput") or hasattr(config, "slaveinput")) @@ -681,7 +670,7 @@ def pytest_runtest_makereport(item, call): ) # On the first failure, ask plugins for a per-failure rerun policy so the protocol - # can give this failure class its own rerun count / semantics and outcome tag. + # can give this failure class its own rerun count / semantics. if result.failed and not hasattr(item, "_rerunfailures_policy"): item._rerunfailures_policy = item.ihook.pytest_rerunfailures_rerun_policy( item=item, report=result, call=call @@ -717,7 +706,6 @@ def pytest_runtest_protocol(item, nextitem): # Track rerun results when all reruns need to pass initial_failure_occurred = False rerun_results = [] # Track result of each rerun (True=passed, False=failed) - rerun_tag = None # From a per-failure rerun policy; stamped on the final failure. policy_applied = False need_to_run = True @@ -728,8 +716,8 @@ def pytest_runtest_protocol(item, nextitem): # Apply a per-failure rerun policy (pytest_rerunfailures_rerun_policy) once the # first attempt has run and makereport has classified its failure. Lets a plugin - # override the rerun count / all-reruns mode for THIS item and tag its final - # failure, without changing how other failures are rerun. + # override the rerun count / all-reruns mode for THIS item, without changing how + # other failures are rerun. if not policy_applied: policy = getattr(item, "_rerunfailures_policy", None) if policy is not None: @@ -739,7 +727,6 @@ def pytest_runtest_protocol(item, nextitem): db.set_test_reruns(item.nodeid, reruns) if policy.all_reruns_need_to_pass is not None: all_reruns_need_to_pass = policy.all_reruns_need_to_pass - rerun_tag = policy.tag for report in reports: # 3 reports: setup, call, teardown report.rerun = item.execution_count - 1 @@ -779,7 +766,6 @@ def pytest_runtest_protocol(item, nextitem): if report.when == "call": report.outcome = "failed" # Log the final report - _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # Not the last rerun yet @@ -795,7 +781,6 @@ def pytest_runtest_protocol(item, nextitem): break # trigger rerun else: # For setup/teardown, just log normally - _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # Normal rerun behavior @@ -803,7 +788,6 @@ def pytest_runtest_protocol(item, nextitem): if should_not_rerun: # last run or no failure detected, log normally - _apply_rerun_tag(report, rerun_tag) item.ihook.pytest_runtest_logreport(report=report) else: # failure detected and reruns not exhausted diff --git a/src/pytest_rerunfailures_newhooks.py b/src/pytest_rerunfailures_newhooks.py index a08fa9c..2947d2c 100644 --- a/src/pytest_rerunfailures_newhooks.py +++ b/src/pytest_rerunfailures_newhooks.py @@ -17,8 +17,7 @@ def pytest_rerunfailures_rerun_policy(item, report, call): sequence -- it is not recomputed on later attempts, so a differently-classed failure on a rerun inherits this policy (the rerun count / mode is fixed by the first failure). Lets a plugin give a specific failure class (e.g. provider-infra errors) its own rerun count / - semantics and an outcome tag, without affecting how other failures are rerun. The first - non-None result wins. + semantics, without affecting how other failures are rerun. The first non-None result wins. Args: item: The test item that failed. diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 7075502..da21d22 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1768,12 +1768,12 @@ def pytest_rerunfailures_rerun_policy(item, report, call): def test_rerun_policy_recovers_on_single_rerun(testdir): - """A tagged failure that recovers on its one rerun passes under all-reruns mode.""" + """A failure that recovers on its one rerun passes under all-reruns mode.""" testdir.makeconftest( """ from pytest_rerunfailures import RerunPolicy def pytest_rerunfailures_rerun_policy(item, report, call): - return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) """ ) testdir.makepyfile( @@ -1799,7 +1799,7 @@ def test_rerun_policy_overrides_all_reruns_need_to_pass(testdir): """ from pytest_rerunfailures import RerunPolicy def pytest_rerunfailures_rerun_policy(item, report, call): - return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) """ ) testdir.makepyfile("def test_fail(): raise Exception('always down')") @@ -1815,7 +1815,7 @@ def test_rerun_policy_only_applies_to_matching_failures(testdir): from pytest_rerunfailures import RerunPolicy def pytest_rerunfailures_rerun_policy(item, report, call): if call.excinfo is not None and call.excinfo.typename == "InfraError": - return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) return None """ ) @@ -1836,27 +1836,6 @@ def test_behavioral(): assert_outcomes(result, passed=0, failed=2, rerun=4) -def test_rerun_policy_tags_final_failure(testdir): - """A policy tag is stamped on the final failed report as ``report.rerun_tag``.""" - testdir.makeconftest( - """ - from pytest_rerunfailures import RerunPolicy - def pytest_rerunfailures_rerun_policy(item, report, call): - return RerunPolicy(reruns=1, tag="infra") - def pytest_runtest_logreport(report): - if report.when == "call" and report.failed: - tag = getattr(report, "rerun_tag", "") - if tag: - import py - py.path.local(__file__).dirpath().join("tag.res").write(tag) - """ - ) - testdir.makepyfile("def test_fail(): raise Exception('boom')") - result = testdir.runpytest("--reruns", "1") - assert_outcomes(result, passed=0, failed=1, rerun=1) - assert testdir.tmpdir.join("tag.res").read() == "infra" - - @pytest.mark.skipif(not has_xdist, reason="requires pytest-xdist") def test_rerun_policy_under_xdist(testdir): """The policy override + recovery work under xdist (policy per-item).""" @@ -1864,7 +1843,7 @@ def test_rerun_policy_under_xdist(testdir): """ from pytest_rerunfailures import RerunPolicy def pytest_rerunfailures_rerun_policy(item, report, call): - return RerunPolicy(reruns=1, all_reruns_need_to_pass=False, tag="infra") + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) """ ) testdir.makepyfile(