From 99ef809df81f3c9659aa5ce88fb1aea5583de9b9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 27 Jul 2026 23:12:05 -0400 Subject: [PATCH] feat(ci): sweep every default branch we own, file one issue Three repositories showed `Release and PyPI Publish - failing` at the top of a PUBLIC repository page for five months and nobody acted. The signal was never hidden; it was ignored. A fourth per-repository badge would repeat that. Separately, no cross-repository detector exists. Four repositories carry a `release-health` workflow (openadapt-flow, openadapt-capture, openadapt-desktop, OpenAdapt) but each `.github/release-health.json` hardcodes a single `"repository"` and asks a different question: is releasable work unpublished? None reads another repository's Actions API. `bin/oa-green` in the workspace asks exactly the right question and is a local script nothing runs on a schedule, so it only helps whoever remembers to run it. This adds one daily job here, because this repository already owns `repos.yml` and already files and updates one issue on a cron in published-version-claims.yml. One run a day, one issue, one place to look. Classification is ported from bin/oa-green, not re-derived: `cancelled`, `timed_out`, `startup_failure` and "still running" are not green; a red newest run is interrogated (deleted / disabled / no longer push-gated -> retired, frozen history, never reported as a failure); a green run on an older head SHA is a note. Two corrections to oa-green, both from evidence: - Retirement only excuses a failure that PREDATES it. oa-green documents this limit itself. openadapt-grounding, openadapt-retrieval and openadapt-viewer failed a push-triggered release at 2026-07-28T00:04Z and were switched to workflow_dispatch at 02:22Z; trusting the current trigger alone would call all three permanently stale and would also miss the next dispatched failure. - An in-flight run gets a grace period. oa-green is a point-in-time check, so any in-flight run correctly fails it. A daily cron alerting on every in-flight run would flap for a reason that is not a defect. Repository list is the live organisation listing, not repos.yml: repos.yml is the docs-site registry and deliberately omits internal tooling, so roughly 20 repositories we own -- including this one -- are absent from it. Archived repositories are excluded (frozen history) and forks are excluded (a red branch on an upstream we forked is not ours to fix). No new secret. A public repository's Actions API is readable unauthenticated, so the default repository-scoped token covers all 26 public repositories. The 8 private ones return 404 and are listed as "not readable", never as failures; an optional OA_SWEEP_TOKEN with actions:read covers them if it is ever set. Cost: standard library only, no dependency install, ~72 seconds and ~160 API reads a day against a 1000/hour budget. This repository is public, so the Actions minutes are free. Verified against the live organisation today: 34 repositories swept, one genuinely not green (openadapt-wright `Deploy Worker`). --- .github/workflows/default-branch-sweep.yml | 138 +++++ scripts/sweep_default_branch_ci.py | 588 +++++++++++++++++++++ tests/test_sweep_default_branch_ci.py | 231 ++++++++ 3 files changed, 957 insertions(+) create mode 100644 .github/workflows/default-branch-sweep.yml create mode 100644 scripts/sweep_default_branch_ci.py create mode 100644 tests/test_sweep_default_branch_ci.py diff --git a/.github/workflows/default-branch-sweep.yml b/.github/workflows/default-branch-sweep.yml new file mode 100644 index 0000000..50a7f57 --- /dev/null +++ b/.github/workflows/default-branch-sweep.yml @@ -0,0 +1,138 @@ +name: Default branch sweep + +# One scheduled job that asks, for EVERY repository we own, whether its default +# branch is genuinely green -- and files ONE issue. +# +# Why here, and why one: three repositories showed +# `Release and PyPI Publish - failing` at the top of a PUBLIC repository page +# for five months and nobody acted. The signal was never hidden; it was ignored. +# A fourth per-repository badge would repeat that. Separately, no cross-repository +# detector exists: the four `release-health` workflows each hardcode a single +# `"repository"` in `.github/release-health.json` and ask a different question, +# and `bin/oa-green` is a local script nothing runs on a schedule. This repository +# already owns `repos.yml` and already files and updates one issue on a daily cron +# in published-version-claims.yml, so this copies that pattern. +# +# Cost: one ubuntu-latest runner, standard library only, no dependency install, +# no lockfile, no cache. About 30-60 seconds a day, and this repository is +# public, so those minutes are free. Roughly 160 API reads against a 1000/hour +# budget, with an explicit per_page on every list call and no pagination loop +# except the organisation listing. + +on: + schedule: + # 07:11 UTC: after the overnight release and Dependabot traffic has settled, + # and offset from published-version-claims.yml at 06:41 so the two scheduled + # jobs in this repository do not start together. + - cron: '11 7 * * *' + workflow_dispatch: + pull_request: + paths: + - 'scripts/sweep_default_branch_ci.py' + - 'tests/test_sweep_default_branch_ci.py' + - '.github/workflows/default-branch-sweep.yml' + +concurrency: + group: default-branch-sweep-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + self-test: + # A detector nobody has seen fire is a detector nobody should trust. The + # offline classification tests also run in Docs CI; running them here keeps + # a change to the detector self-contained. + name: Prove the classifier fires and stays quiet + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true + - run: uv sync --locked --extra dev + - run: uv run pytest tests/test_sweep_default_branch_ci.py -q + + sweep: + name: Sweep every default branch we own + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # A public repository's Actions API is readable with no authentication at + # all, so the default repository-scoped token reads every public + # repository we own and needs no new secret. A private repository returns + # 404 to it and is listed as "not readable", never as a failure. Set the + # optional OA_SWEEP_TOKEN secret to a token with `actions:read` on the + # private repositories to cover them too; the job works without it. + - name: Sweep + id: sweep + env: + GITHUB_TOKEN: ${{ github.token }} + OA_SWEEP_TOKEN: ${{ secrets.OA_SWEEP_TOKEN }} + run: | + python scripts/sweep_default_branch_ci.py \ + --markdown default-branch-sweep.md \ + --github-output "${GITHUB_OUTPUT}" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + + # One issue for the whole organisation, rewritten in place. A new issue + # every day is the same as no issue: it stops being read. Editing a body + # does not notify, so a long-lived gap does not become a daily ping either. + - name: Open, reopen, or update the single sweep issue + if: steps.sweep.outputs.alert == 'true' + env: + GH_TOKEN: ${{ github.token }} + TITLE: "Default branch sweep - a repository we own is not green" + run: | + set -euo pipefail + # Compare the full title in jq rather than searching for it: a colon + # inside a GitHub search phrase is parsed as a qualifier. + EXISTING_JSON=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --limit 1000 --json number,title,state \ + --jq '[.[] | select(.title == env.TITLE)][0] // {}') + EXISTING=$(jq -r '.number // empty' <<< "${EXISTING_JSON}") + EXISTING_STATE=$(jq -r '.state // empty' <<< "${EXISTING_JSON}") + if [ -n "${EXISTING}" ]; then + if [ "${EXISTING_STATE}" = "CLOSED" ]; then + gh issue reopen "${EXISTING}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue edit "${EXISTING}" --repo "${GITHUB_REPOSITORY}" \ + --body-file default-branch-sweep.md + echo "Updated issue #${EXISTING}." + else + gh issue create --repo "${GITHUB_REPOSITORY}" \ + --title "${TITLE}" --body-file default-branch-sweep.md + fi + + # Silence when everything is green. Posting "all green" daily is how an + # alert gets muted. + - name: Close the issue once every default branch is green + if: steps.sweep.outputs.clear == 'true' + env: + GH_TOKEN: ${{ github.token }} + TITLE: "Default branch sweep - a repository we own is not green" + run: | + set -euo pipefail + EXISTING=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state open \ + --limit 1000 --json number,title \ + --jq '[.[] | select(.title == env.TITLE)][0].number // empty') + if [ -n "${EXISTING}" ]; then + gh issue close "${EXISTING}" --repo "${GITHUB_REPOSITORY}" \ + --comment "Every default branch we own is genuinely green as of ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}. Closing automatically." + else + echo "Nothing is red and no issue is open." + fi diff --git a/scripts/sweep_default_branch_ci.py b/scripts/sweep_default_branch_ci.py new file mode 100644 index 0000000..b7e92be --- /dev/null +++ b/scripts/sweep_default_branch_ci.py @@ -0,0 +1,588 @@ +#!/usr/bin/env python3 +"""Sweep every repository we own for a default branch that is not green. + +WHY THIS EXISTS +--------------- +Three repositories published a red ``Release and PyPI Publish`` badge at the top +of a PUBLIC repository page for five months and nobody acted. The signal was +never hidden; it was ignored. Adding a fourth per-repository signal would repeat +that. What is missing is one place that looks at ALL of them, once a day, and +files ONE issue. + +Nothing else does this. Four repositories carry a ``release-health`` workflow +(``openadapt-flow``, ``openadapt-capture``, ``openadapt-desktop``, +``OpenAdapt``), but each ``.github/release-health.json`` hardcodes a single +``"repository"`` and answers a different question: is releasable work +unpublished? None of them reads another repository's Actions API, and none of +them asks whether the default branch is green. ``bin/oa-green`` in the workspace +asks exactly the right question, but it is a local script and nothing runs it on +a schedule -- so it only helps whoever remembers to run it. + +This repository is the right home because it already owns ``repos.yml``, the only +committed multi-repository list, and already files and updates one issue on a +daily cron in ``published-version-claims.yml``. This copies that pattern. + +WHAT COUNTS AS GREEN +-------------------- +Ported from ``bin/oa-green``, not re-derived: + +- ``cancelled``, ``timed_out``, ``startup_failure`` and "still running" are NOT + green. A cancelled run proves nothing about the tree, and concurrency + ``cancel-in-progress`` makes cancellations common exactly when several merges + land together, which is exactly when nobody has attention to spare. +- A red newest run is interrogated before it is believed. GitHub keeps a + workflow's runs for ever, so a workflow that stopped running leaves its last + failure as "the newest run" permanently:: + + workflow file gone from the default branch -> RETIRED + workflow disabled -> RETIRED + no push / pull_request trigger left -> NOT PUSH-GATED + otherwise -> genuinely NOT GREEN + +- A GREEN run whose head SHA is not the current head tested an older tree. That + is worth saying and is never a failure. + +RETIRED and NOT PUSH-GATED are never reported as failures. A daily issue that +cries wolf gets muted, and a muted alert is worse than none -- that is the same +mechanism that let a red badge sit for five months. + +TWO CORRECTIONS TO ``oa-green`` +------------------------------- +1. **Retirement only excuses a failure that PREDATES it.** ``oa-green`` says so + itself: "switching a workflow to workflow_dispatch quiets this tool about + it". That is not hypothetical. ``openadapt-grounding``, + ``openadapt-retrieval`` and ``openadapt-viewer`` failed + ``Release and PyPI Publish`` on ``main`` from 2026-02-18, kept failing for + five months, failed again on a push at 2026-07-28T00:04Z, and were switched + to ``workflow_dispatch``-only at 2026-07-28T02:22Z. A rule that trusts the + current trigger alone would call all three "stale" for ever and would also + stay silent if the next dispatched release failed. + + So a retired or dispatch-only workflow's red run is frozen history **only if + the run started before the commit that retired it**. A red run that started + after that commit happened under the current configuration and is live. The + comparison costs one extra read, and only for a workflow that is already red. + +2. **An in-flight run gets a grace period.** ``oa-green`` is a point-in-time + check a human runs before acting, so any in-flight run correctly fails it. A + daily cron alerting on every in-flight run would flap for a reason that is + not a defect, so an in-progress run only counts once it has been running + longer than ``--running-grace-hours`` (default 6, comfortably longer than the + longest matrix in the organisation). A younger one is context, not an alert. + +Two smaller rules keep the report readable. A ``dynamic/`` workflow path is +GitHub's own synthesised workflow (Dependabot updates, CodeQL default setup): it +has no file in the repository and is never push-gated, so it is neither retired +nor stale-green, and a red one is live. And a stale green run is only worth +saying about a workflow that is actually push-gated -- a nightly or Dependabot +workflow is *supposed* to have last run on an older commit, and reporting each +one buries the handful that mean something. + +REPOSITORY LIST +--------------- +The live organisation listing, not ``repos.yml``. ``repos.yml`` is the docs-site +registry: 13 entries that deliberately omit internal tooling "so a product +evaluator sees the product, not the toolshed". It is correct for its purpose and +wrong for this one -- it omits roughly 20 repositories we own, including this +one. A sweep built on it would leave uncovered precisely the repositories that +are covered by nothing today. + +Archived repositories are excluded: their history is frozen and cannot be +repaired. Forks are excluded: their default branch tracks somebody else's tree, +and AGENTS.md is explicit that a red branch on an upstream we forked is not our +business ("write access is not authority"). + +PERMISSIONS +----------- +A public repository's Actions API is readable with no authentication at all, so +the default repository-scoped ``GITHUB_TOKEN`` reads every public repository we +own. A private repository returns 404 to that token. Private repositories are +therefore listed as "not readable" and never reported as failures, unless a +token with ``actions:read`` on them is supplied in ``OA_SWEEP_TOKEN``. + +COST +---- +About 3 API reads per repository plus one per red workflow: roughly 110 reads +for the whole organisation against a 1000/hour budget. Every list call sets an +explicit ``per_page`` and there is no pagination loop except the organisation +listing itself. Standard library only: no dependency install, no lockfile, no +cache, so the job is a runner start plus a few seconds. + +Usage:: + + python scripts/sweep_default_branch_ci.py + [--markdown FILE] [--github-output FILE] [--run-url URL] + [--running-grace-hours N] +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone + +API_ROOT = "https://api.github.com" +ORG = "OpenAdaptAI" +HTTP_TIMEOUT_SECONDS = 45 + +#: A run with one of these conclusions tested the tree and the tree passed. +#: Everything else -- including ``cancelled``, ``timed_out`` and +#: ``startup_failure`` -- proves nothing about the tree. +GREEN_CONCLUSIONS = frozenset({"success", "neutral", "skipped"}) + +#: Outcomes of :func:`classify_run`. +GREEN = "green" +STALE_GREEN = "stale-green" +RETIRED = "retired" +NOT_PUSH_GATED = "not-push-gated" +IN_FLIGHT = "in-flight" +FAILING = "failing" + +#: Never reported as a failure. Frozen history, not a live signal. +NOT_A_FAILURE = frozenset({GREEN, STALE_GREEN, RETIRED, NOT_PUSH_GATED, IN_FLIGHT}) + + +class Reader: + """Counted, failure-tolerant GitHub reads. + + A repository the token cannot see returns ``None`` rather than raising, so + one private repository can never fail the whole sweep. + """ + + def __init__(self, token: str | None) -> None: + self.token = token + self.calls = 0 + + def get(self, path: str) -> object | None: + self.calls += 1 + request = urllib.request.Request(f"{API_ROOT}/{path}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("X-GitHub-Api-Version", "2022-11-28") + request.add_header("User-Agent", "openadapt-default-branch-sweep") + if self.token: + request.add_header("Authorization", f"Bearer {self.token}") + try: + with urllib.request.urlopen(request, timeout=HTTP_TIMEOUT_SECONDS) as response: + return json.loads(response.read().decode()) + except urllib.error.HTTPError as error: + if error.code in (401, 403, 404, 451): + return None + raise + + +_ON_KEY = re.compile(r"^(on|\"on\"|'on'|True|true)\s*:") +_TRIGGER = re.compile(r"\b(push|pull_request|pull_request_target|merge_group)\b") + + +def is_push_gated(source: str) -> bool: + """Does this workflow still run on a change to the default branch? + + Text-level, because the standard library has no YAML parser and this job + installs nothing. The ``on:`` block is located and read to the next + top-level key. YAML 1.1 turns the bare key ``on`` into ``true``, which some + formatters emit, so that spelling is accepted too. + + An unparseable file returns ``True``: assuming a workflow is live keeps a + real failure visible, whereas assuming it is retired would hide one. + """ + lines = [line for line in source.splitlines() if not line.lstrip().startswith("#")] + block: list[str] = [] + inside = False + for line in lines: + if not inside: + if _ON_KEY.match(line): + inside = True + block.append(line.split(":", 1)[1]) + continue + if line.strip() and not line[0].isspace(): + break + block.append(line) + if not inside: + return True + return _TRIGGER.search("\n".join(block)) is not None + + +def _parse_time(stamp: str) -> datetime: + return datetime.fromisoformat(stamp.replace("Z", "+00:00")) + + +def is_github_managed(path: str) -> bool: + """Is this one of GitHub's own synthesised workflows? + + Dependabot updates and CodeQL default setup run under a ``dynamic/`` path + with no file in the repository. They are never push-gated and can never be + retired by a commit, so the file-based questions do not apply to them. + """ + return path.startswith("dynamic/") + + +def classify_run( + run: dict, + workflow: dict | None, + workflow_source: str | None, + workflow_changed_at: str | None, + head_sha: str, + now: datetime, + running_grace_hours: float, +) -> tuple[str, str]: + """Decide what the newest run of one workflow proves. Pure; no I/O. + + ``workflow`` is the live inventory entry, or ``None`` when the workflow no + longer exists. ``workflow_source`` is the decoded workflow file on the + default branch, or ``None`` when it is not there. ``workflow_changed_at`` is + the timestamp of the newest commit touching that path, used to decide + whether a retirement predates the failure it would excuse. + + Returns ``(outcome, explanation)``. + """ + sha = (run.get("head_sha") or "")[:7] + started = _parse_time(run["created_at"]) + + if run.get("status") != "completed": + hours = (now - started).total_seconds() / 3600 + state = run.get("status", "queued") + if hours >= running_grace_hours: + return FAILING, f"{state} for {hours:.0f}h on `{sha}` -- stuck, and not evidence" + return IN_FLIGHT, f"{state} for {hours:.1f}h on `{sha}` -- not yet evidence either way" + + conclusion = run.get("conclusion") or "none" + path = (workflow or {}).get("path", run.get("path", "")) + managed = is_github_managed(path) + + if conclusion in GREEN_CONCLUSIONS: + stale = bool(head_sha) and run.get("head_sha") != head_sha + # Only meaningful for a workflow that is meant to run on every change. + # A nightly or Dependabot workflow is supposed to have last run on an + # older commit, and saying so about each one buries the ones that matter. + gated = ( + not managed and workflow_source is not None and is_push_gated(workflow_source) + ) + if stale and gated: + return STALE_GREEN, ( + f"{conclusion}, but it tested `{sha}` and head is `{head_sha[:7]}` -- " + "that pass is evidence about a tree nobody runs any more" + ) + return GREEN, conclusion + + # Red. GitHub keeps this run for ever, so establish whether the workflow can + # still run before believing that the tree is broken. + if managed: + # GitHub manages the trigger; there is nothing to retire and no file to + # read. A red one is live. + return FAILING, f"newest run concluded {conclusion} on `{sha}`" + + if workflow is not None and workflow.get("state") not in (None, "active"): + # A disabled workflow cannot run at all, so this run is necessarily past. + return RETIRED, f"workflow {workflow['state']}; the {conclusion} run is frozen history" + + retirement: str | None = None + if workflow is None: + retirement = f"workflow deleted; the {conclusion} run is kept for ever" + elif workflow_source is None: + retirement = ( + f"{workflow['path']} is not on the default branch; " + f"the {conclusion} run is frozen history" + ) + elif not is_push_gated(workflow_source): + retirement = ( + f"dispatch or schedule only, so the {conclusion} run says nothing about the tree" + ) + + if retirement is None: + detail = f"newest run concluded {conclusion} on `{sha}`" + if workflow_changed_at and started < _parse_time(workflow_changed_at): + # Still red, because no green run exists -- but say that the + # configuration has moved on, so the reader re-runs rather than + # re-diagnosing. A path-filtered workflow can sit like this for + # months without another push reaching it. + detail += ( + f"; the workflow file changed afterwards ({workflow_changed_at}), " + "so re-run it to find out whether the fix took" + ) + return FAILING, detail + + # Retirement only excuses a failure that predates it. Three repositories + # failed a push-triggered release at 2026-07-28T00:04Z and were switched to + # workflow_dispatch at 02:22Z; trusting the current trigger alone would have + # hidden the next dispatched failure too. + if workflow_changed_at and started > _parse_time(workflow_changed_at): + return FAILING, ( + f"newest run concluded {conclusion} on `{sha}` AFTER the workflow was " + f"retired ({workflow_changed_at}), so it failed as currently configured" + ) + if workflow_changed_at: + retirement += f" (retired {workflow_changed_at}, after that run)" + return (NOT_PUSH_GATED if workflow is not None and workflow_source else RETIRED), retirement + + +def owned_repositories(reader: Reader) -> list[dict]: + """Every non-archived, non-fork repository in the organisation.""" + found: list[dict] = [] + page = 1 + while True: + batch = reader.get(f"orgs/{ORG}/repos?per_page=100&type=all&page={page}") + if not batch: + break + found.extend(batch) + if len(batch) < 100: + break + page += 1 + return sorted( + (repo for repo in found if not repo.get("archived") and not repo.get("fork")), + key=lambda repo: repo["name"].lower(), + ) + + +def newest_run_per_workflow( + reader: Reader, slug: str, branch: str, workflow_ids: list[int] +) -> dict[int, dict]: + """Newest run per workflow on ``branch``: one list call plus targeted top-ups. + + One very chatty workflow can monopolise the combined feed and silently hide + another workflow's red run. Any active workflow missing from the feed gets + one single-run query, so the extra cost is bounded by the size of that gap + rather than paid once per workflow. + """ + feed = reader.get( + f"repos/{slug}/actions/runs?branch={branch}&per_page=100&exclude_pull_requests=true" + ) + newest: dict[int, dict] = {} + for run in (feed or {}).get("workflow_runs", []): + workflow_id = run["workflow_id"] + if workflow_id not in newest or run["created_at"] > newest[workflow_id]["created_at"]: + newest[workflow_id] = run + + for workflow_id in workflow_ids: + if workflow_id in newest: + continue + extra = reader.get( + f"repos/{slug}/actions/workflows/{workflow_id}/runs" + f"?branch={branch}&per_page=1&exclude_pull_requests=true" + ) + for run in (extra or {}).get("workflow_runs", []): + newest[workflow_id] = run + return newest + + +def sweep_repository(reader: Reader, repo: dict, running_grace_hours: float) -> dict: + slug = repo["full_name"] + branch = repo.get("default_branch") or "main" + result: dict = { + "repo": slug, + "branch": branch, + "failures": [], + "notes": [], + "unreadable": False, + } + + inventory = reader.get(f"repos/{slug}/actions/workflows?per_page=100") + if inventory is None: + result["unreadable"] = True + return result + workflows = {entry["id"]: entry for entry in inventory.get("workflows", [])} + + active_ids = [wid for wid, entry in workflows.items() if entry.get("state") == "active"] + newest = newest_run_per_workflow(reader, slug, branch, active_ids) + if not newest: + return result + + head = reader.get(f"repos/{slug}/commits/{branch}") + head_sha = (head or {}).get("sha", "") + now = datetime.now(timezone.utc) + + source_cache: dict[str, str | None] = {} + changed_cache: dict[str, str | None] = {} + + def workflow_source(path: str) -> str | None: + if path not in source_cache: + contents = reader.get(f"repos/{slug}/contents/{path}?ref={branch}") + if contents is None: + source_cache[path] = None + else: + try: + source_cache[path] = base64.b64decode(contents.get("content", "")).decode( + "utf-8", "replace" + ) + except Exception: + source_cache[path] = "" + return source_cache[path] + + def workflow_changed_at(path: str) -> str | None: + if path not in changed_cache: + commits = reader.get(f"repos/{slug}/commits?path={path}&sha={branch}&per_page=1") + changed_cache[path] = ( + commits[0]["commit"]["committer"]["date"] if commits else None + ) + return changed_cache[path] + + for workflow_id, run in sorted(newest.items(), key=lambda item: item[1]["name"]): + workflow = workflows.get(workflow_id) + path = (workflow or {}).get("path", run.get("path", "")) + completed = run.get("status") == "completed" + conclusion = run.get("conclusion") or "none" + red = completed and conclusion not in GREEN_CONCLUSIONS + stale_green = ( + completed + and conclusion in GREEN_CONCLUSIONS + and bool(head_sha) + and run.get("head_sha") != head_sha + ) + + # Reads are spent only where the answer can change the outcome: a red + # run, or a green run on an older tree that may be worth a note. An + # in-flight or up-to-date run costs nothing extra. + source: str | None = None + changed_at: str | None = None + if (red or stale_green) and not is_github_managed(path) and path: + if workflow is not None and workflow.get("state") == "active": + source = workflow_source(path) + if red: + changed_at = workflow_changed_at(path) + + outcome, explanation = classify_run( + run, workflow, source, changed_at, head_sha, now, running_grace_hours + ) + if outcome == GREEN: + continue + entry = { + "workflow": run["name"], + "outcome": outcome, + "explanation": explanation, + "url": run.get("html_url", ""), + } + if outcome == FAILING: + result["failures"].append(entry) + else: + result["notes"].append(entry) + + return result + + +def render(results: list[dict], reader: Reader, run_url: str, running_grace_hours: float) -> str: + failing = sorted((r for r in results if r["failures"]), key=lambda r: r["repo"]) + noted = sorted( + (r for r in results if not r["failures"] and r["notes"]), key=lambda r: r["repo"] + ) + unreadable = sorted((r for r in results if r["unreadable"]), key=lambda r: r["repo"]) + + lines = [ + f"**{len(failing)} of {len(results)} repositories we own do not have a genuinely " + "green default branch.**", + "", + "`cancelled`, `timed_out` and `startup_failure` are not green: such a run proves " + "nothing about the tree. The usual cause is concurrency `cancel-in-progress` when " + "several merges land at once, so re-run the cancelled run rather than assuming the " + "older success still stands.", + "", + "This issue is rewritten in place once a day and closed when everything is green. " + "Editing a body does not notify, so a long-lived gap does not become a daily ping.", + "", + ] + + for result in failing: + lines.append( + f"### [{result['repo']}](https://github.com/{result['repo']}) " + f"(`{result['branch']}`)" + ) + lines.append("") + for entry in result["failures"]: + link = f" ([run]({entry['url']}))" if entry["url"] else "" + lines.append(f"- **{entry['workflow']}** — {entry['explanation']}{link}") + for entry in result["notes"]: + lines.append(f"- _{entry['workflow']}: {entry['outcome']} — {entry['explanation']}_") + lines.append("") + + if noted: + lines += [ + "
Green, with notes — none of these is a failure", + "", + "`retired` and `not-push-gated` entries are frozen history: GitHub keeps a " + "workflow's last run for ever, so a workflow that stopped running would " + "otherwise be reported red permanently. `stale-green` passed on an older tree. " + f"`in-flight` has been running for less than {running_grace_hours:g}h.", + "", + ] + for result in noted: + lines.append(f"- **{result['repo']}**") + for entry in result["notes"]: + lines.append( + f" - {entry['workflow']}: {entry['outcome']} — {entry['explanation']}" + ) + lines += ["", "
", ""] + + if unreadable: + lines += [ + "
Not readable by this token — skipped, not failed", + "", + "A private repository's Actions API returns 404 to the repository-scoped " + "`GITHUB_TOKEN`. Set an `OA_SWEEP_TOKEN` secret holding a token with " + "`actions:read` on these repositories to include them.", + "", + ] + lines += [f"- {result['repo']}" for result in unreadable] + lines += ["", "
", ""] + + lines.append( + f"Swept {len(results)} repositories in {reader.calls} API reads. {run_url}".strip() + ) + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--markdown", help="write the issue body to this file") + parser.add_argument("--github-output", help="append alert= and clear= to this file") + parser.add_argument("--run-url", default="", help="link back to the run that produced this") + parser.add_argument( + "--running-grace-hours", + type=float, + default=6.0, + help="how long an in-progress run may run before it counts as stuck", + ) + args = parser.parse_args(argv) + + token = ( + os.environ.get("OA_SWEEP_TOKEN") + or os.environ.get("GH_TOKEN") + or os.environ.get("GITHUB_TOKEN") + ) + reader = Reader(token) + + repositories = owned_repositories(reader) + if not repositories: + print( + f"sweep: the {ORG} repository listing returned nothing; refusing to report " + "'everything is green' from an empty list", + file=sys.stderr, + ) + return 2 + + results = [sweep_repository(reader, repo, args.running_grace_hours) for repo in repositories] + failing = [result for result in results if result["failures"]] + body = render(results, reader, args.run_url, args.running_grace_hours) + + print(body) + print( + f"\nswept {len(results)} repositories in {reader.calls} API reads; " + f"{len(failing)} not green", + file=sys.stderr, + ) + + if args.markdown: + with open(args.markdown, "w", encoding="utf-8") as handle: + handle.write(body + "\n") + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as handle: + handle.write(f"alert={'true' if failing else 'false'}\n") + handle.write(f"clear={'false' if failing else 'true'}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_sweep_default_branch_ci.py b/tests/test_sweep_default_branch_ci.py new file mode 100644 index 0000000..d207ffa --- /dev/null +++ b/tests/test_sweep_default_branch_ci.py @@ -0,0 +1,231 @@ +"""The cross-repository sweep must fire on a real red branch and stay quiet otherwise. + +Proving a detector passes today is worthless on its own. These tests prove it +FAILS on the exact conditions it exists to catch -- including the real incident +that motivated it -- and that it stays quiet on the conditions that would make it +noise, because a daily issue that cries wolf gets muted, and a muted alert is +worse than none. + +The 2026-07-28 incident, reproduced below as +``test_reports_a_failure_that_happened_after_the_retirement``: +``openadapt-grounding``, ``openadapt-retrieval`` and ``openadapt-viewer`` failed +``Release and PyPI Publish`` on ``main`` from 2026-02-18, kept failing for five +months, failed again on a push at 00:04Z, and were switched to +``workflow_dispatch``-only at 02:22Z. Trusting the current trigger alone would +call all three permanently stale and would also miss the next dispatched failure. +""" + +import pathlib +import sys +from datetime import datetime, timezone + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from sweep_default_branch_ci import ( # noqa: E402 + FAILING, + GREEN, + IN_FLIGHT, + NOT_PUSH_GATED, + RETIRED, + STALE_GREEN, + classify_run, + is_github_managed, + is_push_gated, +) + +NOW = datetime(2026, 7, 28, 12, 0, tzinfo=timezone.utc) +HEAD = "a" * 40 +OLD = "b" * 40 + +PUSH_GATED = "name: CI\non:\n push:\n branches: [main]\n pull_request:\njobs: {}\n" +DISPATCH_ONLY = "name: Release\non:\n workflow_dispatch:\njobs: {}\n" +SCHEDULE_ONLY = "name: Nightly\non:\n schedule:\n - cron: '0 3 * * *'\njobs: {}\n" + +ACTIVE = {"id": 1, "state": "active", "path": ".github/workflows/ci.yml"} + + +def run(**kwargs): + base = { + "name": "CI", + "status": "completed", + "conclusion": "success", + "head_sha": HEAD, + "created_at": "2026-07-28T10:00:00Z", + "path": ".github/workflows/ci.yml", + } + base.update(kwargs) + return base + + +def classify(r, workflow=ACTIVE, source=PUSH_GATED, changed_at=None, head=HEAD, grace=6.0): + return classify_run(r, workflow, source, changed_at, head, NOW, grace)[0] + + +# -------------------------------------------------------------------------- +# It must fire on what is not green +# -------------------------------------------------------------------------- + + +def test_a_failed_run_on_a_live_workflow_is_a_failure(): + assert classify(run(conclusion="failure")) == FAILING + + +def test_cancelled_is_not_green(): + """The reason bin/oa-green exists: 12 of 13 jobs ticked proves nothing.""" + assert classify(run(conclusion="cancelled")) == FAILING + + +def test_timed_out_and_startup_failure_are_not_green(): + assert classify(run(conclusion="timed_out")) == FAILING + assert classify(run(conclusion="startup_failure")) == FAILING + + +def test_a_run_stuck_in_progress_past_the_grace_period_is_a_failure(): + stuck = run(status="in_progress", conclusion=None, created_at="2026-07-27T12:00:00Z") + assert classify(stuck) == FAILING + + +def test_reports_a_failure_that_happened_after_the_retirement(): + """The 2026-07-28 incident. A dispatch-only workflow that failed TODAY is live. + + Switching a workflow to workflow_dispatch must not silence a failure that + happened under that very configuration. + """ + outcome = classify( + run(name="Release and PyPI Publish", conclusion="failure", + created_at="2026-07-28T04:00:00Z"), + source=DISPATCH_ONLY, + changed_at="2026-07-28T02:22:00Z", + ) + assert outcome == FAILING + + +def test_a_failure_predating_a_workflow_edit_stays_red_but_says_so(): + """openadapt-wright: 9 for 9 failures in March, a gate added on 2026-07-28. + + No green run exists, so it stays red. The reader is told to re-run rather + than re-diagnose, because a path-filtered workflow may not be triggered + again for months. + """ + outcome, explanation = classify_run( + run(name="Deploy Worker", conclusion="failure", created_at="2026-03-19T17:59:55Z"), + ACTIVE, PUSH_GATED, "2026-07-28T02:22:22Z", HEAD, NOW, 6.0, + ) + assert outcome == FAILING + assert "re-run it" in explanation + + +def test_a_red_github_managed_workflow_is_a_failure(): + """Dependabot and CodeQL default setup have no file, but they are live.""" + managed = {"id": 9, "state": "active", "path": "dynamic/dependabot/dependabot-updates"} + assert classify(run(conclusion="failure"), workflow=managed, source=None) == FAILING + + +# -------------------------------------------------------------------------- +# It must stay quiet on what is not a failure +# -------------------------------------------------------------------------- + + +def test_a_success_on_head_is_green(): + assert classify(run()) == GREEN + assert classify(run(conclusion="neutral")) == GREEN + assert classify(run(conclusion="skipped")) == GREEN + + +def test_a_deleted_workflow_is_retired_not_failing(): + """GitHub keeps old runs for ever; a deleted workflow's last failure is history.""" + outcome = classify( + run(conclusion="failure", created_at="2026-02-18T04:54:00Z"), + workflow=None, + source=None, + changed_at="2026-07-01T00:00:00Z", + ) + assert outcome == RETIRED + + +def test_a_disabled_workflow_is_retired_not_failing(): + disabled = {"id": 1, "state": "disabled_inactivity", "path": ".github/workflows/x.yml"} + assert classify(run(conclusion="failure"), workflow=disabled) == RETIRED + + +def test_a_dispatch_only_workflow_whose_failure_predates_retirement_is_not_a_failure(): + """The other half of the 2026-07-28 incident: the five-month-old failure itself.""" + outcome = classify( + run(name="Release and PyPI Publish", conclusion="failure", + created_at="2026-07-28T00:04:00Z"), + source=DISPATCH_ONLY, + changed_at="2026-07-28T02:22:00Z", + ) + assert outcome == NOT_PUSH_GATED + + +def test_a_schedule_only_workflow_is_not_push_gated(): + outcome = classify( + run(conclusion="failure", created_at="2026-01-01T00:00:00Z"), + source=SCHEDULE_ONLY, + changed_at="2026-06-01T00:00:00Z", + ) + assert outcome == NOT_PUSH_GATED + + +def test_a_young_in_flight_run_does_not_alert(): + """A run that started 20 minutes ago is not a defect; alerting on it flaps.""" + fresh = run(status="in_progress", conclusion=None, created_at="2026-07-28T11:40:00Z") + assert classify(fresh) == IN_FLIGHT + + +# -------------------------------------------------------------------------- +# A green run on an older tree is a note, and only where it means something +# -------------------------------------------------------------------------- + + +def test_a_green_run_on_an_older_commit_is_reported_as_stale(): + assert classify(run(head_sha=OLD)) == STALE_GREEN + + +def test_a_stale_green_nightly_workflow_is_not_reported(): + """A scheduled workflow is SUPPOSED to have last run on an older commit.""" + assert classify(run(head_sha=OLD), source=SCHEDULE_ONLY) == GREEN + + +def test_a_stale_green_github_managed_workflow_is_not_reported(): + managed = {"id": 9, "state": "active", "path": "dynamic/dependabot/dependabot-updates"} + assert classify(run(head_sha=OLD), workflow=managed, source=None) == GREEN + + +# -------------------------------------------------------------------------- +# Trigger parsing +# -------------------------------------------------------------------------- + + +def test_push_gating_detection(): + assert is_push_gated(PUSH_GATED) + assert not is_push_gated(DISPATCH_ONLY) + assert not is_push_gated(SCHEDULE_ONLY) + assert is_push_gated("on: [push, workflow_dispatch]\njobs: {}\n") + assert is_push_gated("on: push\njobs: {}\n") + assert is_push_gated('"on":\n pull_request:\njobs: {}\n') + # YAML 1.1 turns a bare `on` key into `true`; some formatters emit that. + assert is_push_gated("true:\n push:\njobs: {}\n") + + +def test_a_later_jobs_key_does_not_leak_into_the_on_block(): + """`jobs:` starts at column 0, so the on-block read stops there.""" + source = "on:\n workflow_dispatch:\njobs:\n push:\n runs-on: ubuntu-latest\n" + assert not is_push_gated(source) + + +def test_a_commented_out_trigger_does_not_count(): + assert not is_push_gated("on:\n workflow_dispatch:\n # push:\njobs: {}\n") + + +def test_an_unparseable_workflow_is_assumed_live(): + """Assuming live keeps a real failure visible; assuming retired hides one.""" + assert is_push_gated("this is not a workflow at all") + + +def test_github_managed_paths(): + assert is_github_managed("dynamic/dependabot/dependabot-updates") + assert is_github_managed("dynamic/github-code-scanning/codeql") + assert not is_github_managed(".github/workflows/ci.yml")