From a192caa2d47c2be1ef26c9f66bd7ad649f65ab84 Mon Sep 17 00:00:00 2001 From: stacknil Date: Sat, 20 Jun 2026 13:30:15 +0800 Subject: [PATCH] feat(sbom): add report evidence confidence --- scripts/validate-reviewer-routes.py | 19 +++ .../sbom-diff-and-risk/docs/report-schema.md | 28 ++++- .../sbom-diff-and-risk/docs/reviewer-path.md | 2 + .../docs/summary-json-ci-cookbook.md | 7 ++ .../examples/sample-policy-fail-report.json | 3 + .../examples/sample-policy-fail-report.md | 1 + .../examples/sample-policy-warn-report.json | 3 + .../examples/sample-policy-warn-report.md | 1 + .../examples/sample-provenance-report.json | 3 + .../examples/sample-provenance-report.md | 1 + .../examples/sample-report.json | 5 +- .../examples/sample-report.md | 1 + .../examples/sample-requirements-report.json | 5 +- .../examples/sample-requirements-report.md | 1 + .../examples/sample-scorecard-report.json | 3 + .../examples/sample-scorecard-report.md | 1 + .../examples/sample-summary.json | 3 +- .../src/sbom_diff_risk/evidence_confidence.py | 64 ++++++++++ .../src/sbom_diff_risk/models.py | 9 ++ .../src/sbom_diff_risk/report_json.py | 5 + .../src/sbom_diff_risk/report_md.py | 2 + .../test_cli_no_enrichment_regression.py | 5 + .../tests/test_cli_summary_json.py | 4 + .../tests/test_evidence_confidence.py | 115 ++++++++++++++++++ .../tests/test_provenance_reporting.py | 3 + .../sbom-diff-and-risk/tests/test_reports.py | 7 ++ .../tests/test_scorecard_reporting.py | 3 + 27 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py create mode 100644 tools/sbom-diff-and-risk/tests/test_evidence_confidence.py diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py index 9cac655..571055b 100644 --- a/scripts/validate-reviewer-routes.py +++ b/scripts/validate-reviewer-routes.py @@ -15,6 +15,7 @@ Path("docs/reviewer-brief.md"), Path("docs/repo-scope-map.md"), Path("docs/risk-model-boundary.md"), + Path("tools/sbom-diff-and-risk/docs/report-schema.md"), Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"), Path("projects/precipitation-anomaly-diagnostics/docs/reviewer-path.md"), Path("projects/precipitation-anomaly-diagnostics-lab/docs/reviewer-path.md"), @@ -69,6 +70,13 @@ "tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py", "tools/sbom-diff-and-risk/src/sbom_diff_risk/risk.py", }, + Path("tools/sbom-diff-and-risk/docs/report-schema.md"): { + "tools/sbom-diff-and-risk/docs/policy-decision-ci-cookbook.md", + "tools/sbom-diff-and-risk/docs/policy-decision-explainability.md", + "tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md", + "tools/sbom-diff-and-risk/examples/sample-report.json", + "tools/sbom-diff-and-risk/examples/sample-summary.json", + }, Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"): { ".github/workflows/reviewer-route-contract-ci.yml", "docs/risk-model-boundary.md", @@ -171,6 +179,16 @@ "suspicious_source", "not_evaluated", ), + Path("tools/sbom-diff-and-risk/docs/report-schema.md"): ( + "evidence_confidence", + "local_manifest_only", + "sbom_present", + "policy_matched", + "enrichment_mocked", + "enrichment_live", + "not a package safety verdict", + "not a CVE result", + ), Path("tools/sbom-diff-and-risk/docs/reviewer-path.md"): ( "Artifact evidence map", "Reviewer route contract", @@ -183,6 +201,7 @@ "every tracked reviewer-surface Markdown file is covered", "python scripts/validate-reviewer-routes.py", "No network", + "summary.evidence_confidence", "not current PyPI package truth", "not current repository reputation", "It does not decide whether a dependency is safe.", diff --git a/tools/sbom-diff-and-risk/docs/report-schema.md b/tools/sbom-diff-and-risk/docs/report-schema.md index 404f6e1..78b26b9 100644 --- a/tools/sbom-diff-and-risk/docs/report-schema.md +++ b/tools/sbom-diff-and-risk/docs/report-schema.md @@ -15,7 +15,8 @@ JSON reports currently use this top-level structure: | Field | Description | | --- | --- | -| `summary` | Compact count-only run summary for deterministic machine consumption. | +| `summary` | Compact run summary for deterministic machine consumption. | +| `evidence_confidence` | Highest evidence-confidence level represented by this report. | | `components` | Added, removed, and changed component records. | | `risks` | Heuristic risk findings generated from the diff. | | `policy_evaluation` | Policy evaluation details when policy state is represented in the report. | @@ -28,7 +29,7 @@ JSON reports currently use this top-level structure: | `scorecard_summary` | OpenSSF Scorecard evidence summary when available from report presentation. | | `enrichment_metadata` | Top-level enrichment metadata used by trust-signal report sections. | | `trust_signal_notes` | Review notes for provenance and Scorecard trust signals. | -| `metadata` | Run metadata such as input formats, generation time, strict mode, policy state, and enrichment state. | +| `metadata` | Run metadata such as input formats, generation time, strict mode, policy state, evidence confidence, and enrichment state. | | `notes` | Additional report notes. | When provenance policy fields are relevant, reports may also include @@ -94,8 +95,9 @@ locks the standalone policy sidecar shape for a strict policy example. ## Summary contract `summary` is the stable, compact entry point for automation that needs counts -without walking the full report. The `--summary-json PATH` CLI option writes -only this stable `report.json["summary"]` object. +and evidence-level labels without walking the full report. The +`--summary-json PATH` CLI option writes only this stable +`report.json["summary"]` object. The checked-in [../examples/sample-summary.json](../examples/sample-summary.json) artifact is the summary-only output for the default CycloneDX example and @@ -112,11 +114,25 @@ Base `summary` fields: | `removed` | Number of components present only in the before input. | | `changed` | Number of components present in both inputs with a detected change. | | `risk_counts` | Map of risk bucket name to count. | +| `evidence_confidence` | Highest evidence-confidence level represented by this report. | There is intentionally no `unchanged` field. The current diff model does not track unchanged components, so reporting an unchanged count would imply a model guarantee that does not exist. +`evidence_confidence` is a reviewer-facing evidence label. It explains what +kind of evidence the report contains; it is not a package safety verdict and is +not a CVE result. The same value appears at top level, in `summary`, and in +`metadata.evidence_confidence`. + +| Value | Meaning | +| --- | --- | +| `local_manifest_only` | The report was produced from local manifest-style inputs without SBOM input, policy matches, or enrichment evidence. | +| `sbom_present` | At least one input is an SBOM format such as CycloneDX JSON or SPDX JSON. | +| `policy_matched` | Local policy evaluation produced at least one blocking, warning, or suppressed policy match. | +| `enrichment_mocked` | Enrichment-shaped evidence is present without recorded live network access, or the report explicitly marks constructed snapshot evidence as mocked. | +| `enrichment_live` | Opt-in enrichment recorded live network access for PyPI provenance or OpenSSF Scorecard evidence. | + `summary.policy` appears only when a policy is applied. Absence of `summary.policy` means policy was not used, not that policy evaluation failed. @@ -153,7 +169,9 @@ stable for tests and downstream consumers. - The schema is conservative and additive where possible. - Missing `summary.policy` means policy was not applied. - Missing `summary.enrichment` means PyPI and Scorecard enrichment were not used. -- Runtime details remain in the fuller report fields; `summary` stays count-only. +- Runtime details remain in the fuller report fields; `summary` stays compact. +- `evidence_confidence` describes evidence source level only; it does not rank + dependency safety. ## Non-claims diff --git a/tools/sbom-diff-and-risk/docs/reviewer-path.md b/tools/sbom-diff-and-risk/docs/reviewer-path.md index 7dbb7e2..98fb422 100644 --- a/tools/sbom-diff-and-risk/docs/reviewer-path.md +++ b/tools/sbom-diff-and-risk/docs/reviewer-path.md @@ -68,6 +68,8 @@ Then read: Look for these reviewer anchors: - `summary` is the compact machine-readable entry point +- `summary.evidence_confidence` labels the highest evidence level represented + by the report - `summary.policy` appears only when policy evaluation runs - `summary.enrichment` appears only when enrichment evidence exists - policy findings explain `decision_reason`, `policy_rule`, diff --git a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md index 06ba400..4bcdd25 100644 --- a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md +++ b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md @@ -40,9 +40,11 @@ added = summary["added"] removed = summary["removed"] changed = summary["changed"] risk_counts = summary["risk_counts"] +evidence_confidence = summary["evidence_confidence"] print(f"added={added} removed={removed} changed={changed}") print(f"risk_counts={risk_counts}") +print(f"evidence_confidence={evidence_confidence}") max_new_packages = 2 if risk_counts.get("new_package", 0) > max_new_packages: @@ -61,9 +63,11 @@ $added = $summary.added $removed = $summary.removed $changed = $summary.changed $newPackageCount = $summary.risk_counts.new_package +$evidenceConfidence = $summary.evidence_confidence Write-Output "added=$added removed=$removed changed=$changed" Write-Output "new_package=$newPackageCount" +Write-Output "evidence_confidence=$evidenceConfidence" $maxNewPackages = 2 if ($newPackageCount -gt $maxNewPackages) { @@ -75,6 +79,9 @@ if ($newPackageCount -gt $maxNewPackages) { - `summary.policy` appears only when policy evaluation is applied. - `summary.enrichment` appears only when PyPI or Scorecard enrichment is used. +- `summary.evidence_confidence` is always present and can be + `local_manifest_only`, `sbom_present`, `policy_matched`, + `enrichment_mocked`, or `enrichment_live`. - `unchanged` is absent because unchanged components are not modeled. - Absence of `summary.policy` or `summary.enrichment` means the feature was not used, not that it failed. diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json index e212d43..a88a064 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.json @@ -12,6 +12,7 @@ "suspicious_source": 0, "not_evaluated": 2 }, + "evidence_confidence": "policy_matched", "policy": { "status": "fail", "blocking": 3, @@ -19,6 +20,7 @@ "suppressed": 0 } }, + "evidence_confidence": "policy_matched", "components": { "added": [ { @@ -700,6 +702,7 @@ }, "exit_code": 1 }, + "evidence_confidence": "policy_matched", "enrichment": { "mode": "offline_default", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.md b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.md index 4cb165a..1994a82 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-policy-fail-report.md @@ -6,6 +6,7 @@ - Added: 1 - Removed: 0 - Version changes: 1 +- Evidence confidence: policy_matched ## Risk buckets - new_package: 1 diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json index f487337..e790ee7 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.json @@ -12,6 +12,7 @@ "suspicious_source": 0, "not_evaluated": 2 }, + "evidence_confidence": "policy_matched", "policy": { "status": "warn", "blocking": 0, @@ -19,6 +20,7 @@ "suppressed": 0 } }, + "evidence_confidence": "policy_matched", "components": { "added": [ { @@ -553,6 +555,7 @@ }, "exit_code": 0 }, + "evidence_confidence": "policy_matched", "enrichment": { "mode": "offline_default", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.md b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.md index 1bf7a6d..a184feb 100644 --- a/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-policy-warn-report.md @@ -6,6 +6,7 @@ - Added: 1 - Removed: 0 - Version changes: 1 +- Evidence confidence: policy_matched ## Risk buckets - new_package: 1 diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json index 7b0c92b..4aa918a 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json @@ -12,6 +12,7 @@ "suspicious_source": 0, "not_evaluated": 0 }, + "evidence_confidence": "enrichment_mocked", "policy": { "status": "fail", "blocking": 2, @@ -32,6 +33,7 @@ } } }, + "evidence_confidence": "enrichment_mocked", "components": { "added": [ { @@ -744,6 +746,7 @@ }, "exit_code": 1 }, + "evidence_confidence": "enrichment_mocked", "enrichment": { "mode": "opt_in_pypi", "pypi_enabled": true, diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.md b/tools/sbom-diff-and-risk/examples/sample-provenance-report.md index 963c7ce..294ed7e 100644 --- a/tools/sbom-diff-and-risk/examples/sample-provenance-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-provenance-report.md @@ -6,6 +6,7 @@ - Added: 2 - Removed: 0 - Version changes: 1 +- Evidence confidence: enrichment_mocked ## Risk buckets - new_package: 2 diff --git a/tools/sbom-diff-and-risk/examples/sample-report.json b/tools/sbom-diff-and-risk/examples/sample-report.json index 39a0230..5f9caf6 100644 --- a/tools/sbom-diff-and-risk/examples/sample-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-report.json @@ -11,8 +11,10 @@ "stale_package": 0, "suspicious_source": 0, "not_evaluated": 2 - } + }, + "evidence_confidence": "sbom_present" }, + "evidence_confidence": "sbom_present", "components": { "added": [ { @@ -480,6 +482,7 @@ }, "exit_code": 0 }, + "evidence_confidence": "sbom_present", "enrichment": { "mode": "offline_default", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-report.md b/tools/sbom-diff-and-risk/examples/sample-report.md index 5b73e63..ddfe588 100644 --- a/tools/sbom-diff-and-risk/examples/sample-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-report.md @@ -6,6 +6,7 @@ - Added: 1 - Removed: 0 - Version changes: 1 +- Evidence confidence: sbom_present ## Risk buckets - new_package: 1 diff --git a/tools/sbom-diff-and-risk/examples/sample-requirements-report.json b/tools/sbom-diff-and-risk/examples/sample-requirements-report.json index 71a084a..c0d4ca6 100644 --- a/tools/sbom-diff-and-risk/examples/sample-requirements-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-requirements-report.json @@ -11,8 +11,10 @@ "stale_package": 0, "suspicious_source": 0, "not_evaluated": 2 - } + }, + "evidence_confidence": "local_manifest_only" }, + "evidence_confidence": "local_manifest_only", "components": { "added": [ { @@ -416,6 +418,7 @@ }, "exit_code": 0 }, + "evidence_confidence": "local_manifest_only", "enrichment": { "mode": "offline_default", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-requirements-report.md b/tools/sbom-diff-and-risk/examples/sample-requirements-report.md index 17f3d36..665a6b6 100644 --- a/tools/sbom-diff-and-risk/examples/sample-requirements-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-requirements-report.md @@ -6,6 +6,7 @@ - Added: 1 - Removed: 0 - Version changes: 1 +- Evidence confidence: local_manifest_only ## Risk buckets - new_package: 1 diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json index a77a88a..455259f 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json @@ -12,6 +12,7 @@ "suspicious_source": 0, "not_evaluated": 0 }, + "evidence_confidence": "enrichment_mocked", "policy": { "status": "warn", "blocking": 0, @@ -31,6 +32,7 @@ } } }, + "evidence_confidence": "enrichment_mocked", "components": { "added": [ { @@ -573,6 +575,7 @@ }, "exit_code": 0 }, + "evidence_confidence": "enrichment_mocked", "enrichment": { "mode": "opt_in_scorecard", "pypi_enabled": false, diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md index cf24ba2..d910ba6 100644 --- a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md +++ b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md @@ -6,6 +6,7 @@ - Added: 2 - Removed: 0 - Version changes: 1 +- Evidence confidence: enrichment_mocked ## Risk buckets - new_package: 2 diff --git a/tools/sbom-diff-and-risk/examples/sample-summary.json b/tools/sbom-diff-and-risk/examples/sample-summary.json index 8c255df..dd1e57f 100644 --- a/tools/sbom-diff-and-risk/examples/sample-summary.json +++ b/tools/sbom-diff-and-risk/examples/sample-summary.json @@ -10,5 +10,6 @@ "stale_package": 0, "suspicious_source": 0, "not_evaluated": 2 - } + }, + "evidence_confidence": "sbom_present" } diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py new file mode 100644 index 0000000..77c5f3a --- /dev/null +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from .models import CompareReport, Component, EvidenceConfidence + +_SBOM_FORMATS = {"cyclonedx-json", "spdx-json"} + + +def evidence_confidence_for_report(report: CompareReport) -> EvidenceConfidence: + if report.metadata.evidence_confidence is not None: + return report.metadata.evidence_confidence + + if _has_enrichment_evidence(report): + if _has_live_enrichment(report): + return EvidenceConfidence.ENRICHMENT_LIVE + return EvidenceConfidence.ENRICHMENT_MOCKED + + if _has_policy_match(report): + return EvidenceConfidence.POLICY_MATCHED + + if _has_sbom_input(report): + return EvidenceConfidence.SBOM_PRESENT + + return EvidenceConfidence.LOCAL_MANIFEST_ONLY + + +def evidence_confidence_value(report: CompareReport) -> str: + return evidence_confidence_for_report(report).value + + +def _has_enrichment_evidence(report: CompareReport) -> bool: + metadata = report.metadata.enrichment + if metadata.pypi_enabled or metadata.scorecard_enabled: + return True + + return any(component.provenance is not None or component.scorecard is not None for component in _all_components(report)) + + +def _has_live_enrichment(report: CompareReport) -> bool: + metadata = report.metadata.enrichment + return ( + metadata.network_access_performed + or metadata.pypi_network_access_performed + or metadata.scorecard_network_access_performed + ) + + +def _has_policy_match(report: CompareReport) -> bool: + evaluation = report.metadata.policy_evaluation + if evaluation is None or not evaluation.applied: + return False + + return bool(evaluation.blocking_violations or evaluation.warning_violations or evaluation.suppressed_violations) + + +def _has_sbom_input(report: CompareReport) -> bool: + return report.metadata.before_format in _SBOM_FORMATS or report.metadata.after_format in _SBOM_FORMATS + + +def _all_components(report: CompareReport) -> tuple[Component, ...]: + components: list[Component] = [*report.components.added, *report.components.removed] + for change in report.components.changed: + components.append(change.before) + components.append(change.after) + return tuple(components) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py index 9c399c6..23ca64e 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py @@ -18,6 +18,14 @@ class RiskBucket(StrEnum): NOT_EVALUATED = "not_evaluated" +class EvidenceConfidence(StrEnum): + LOCAL_MANIFEST_ONLY = "local_manifest_only" + SBOM_PRESENT = "sbom_present" + POLICY_MATCHED = "policy_matched" + ENRICHMENT_MOCKED = "enrichment_mocked" + ENRICHMENT_LIVE = "enrichment_live" + + class ProvenanceStatus(StrEnum): PROVENANCE_AVAILABLE = "provenance_available" ATTESTATION_AVAILABLE = "attestation_available" @@ -178,6 +186,7 @@ class ReportMetadata: strict: bool = False stub: bool = True policy_evaluation: PolicyEvaluation | None = None + evidence_confidence: EvidenceConfidence | None = None enrichment: ReportEnrichmentMetadata = field(default_factory=ReportEnrichmentMetadata) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_json.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_json.py index 1c35fdf..3f8994b 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_json.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_json.py @@ -2,6 +2,7 @@ import json +from .evidence_confidence import evidence_confidence_value from .enrichment import enrichment_metadata_to_dict, provenance_evidence_to_dict from .models import CompareReport, Component, ComponentChange, ReportEnrichmentMetadata, RiskFinding from .presentation import build_policy_report_sections, build_trust_signal_report_sections @@ -12,8 +13,10 @@ def render_report_json(report: CompareReport) -> str: policy_sections = build_policy_report_sections(report.metadata.policy_evaluation) trust_signal_sections = build_trust_signal_report_sections(report) + evidence_confidence = evidence_confidence_value(report) payload = { "summary": _summary_to_dict(report), + "evidence_confidence": evidence_confidence, "components": { "added": [_component_to_dict(component) for component in report.components.added], "removed": [_component_to_dict(component) for component in report.components.removed], @@ -37,6 +40,7 @@ def render_report_json(report: CompareReport) -> str: "strict": report.metadata.strict, "stub": report.metadata.stub, "policy_evaluation": policy_sections["policy_evaluation"], + "evidence_confidence": evidence_confidence, "enrichment": enrichment_metadata_to_dict(report.metadata.enrichment), }, "notes": list(report.notes), @@ -78,6 +82,7 @@ def _summary_to_dict(report: CompareReport) -> dict[str, object]: "removed": report.summary.removed, "changed": report.summary.changed, "risk_counts": dict(report.summary.risk_counts), + "evidence_confidence": evidence_confidence_value(report), } policy_summary = _policy_summary_to_dict(report.metadata.policy_evaluation) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_md.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_md.py index 920947c..d67472c 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_md.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/report_md.py @@ -1,6 +1,7 @@ from __future__ import annotations from .diffing import component_key +from .evidence_confidence import evidence_confidence_value from .models import CompareReport from .presentation import ( build_trust_signal_report_sections, @@ -30,6 +31,7 @@ def render_report_markdown(report: CompareReport) -> str: f"- Added: {report.summary.added}", f"- Removed: {report.summary.removed}", f"- Version changes: {report.summary.changed}", + f"- Evidence confidence: {evidence_confidence_value(report)}", "", "## Risk buckets", ] diff --git a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py index 6fdc27c..588b4ea 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py @@ -59,6 +59,9 @@ def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 assert payload["metadata"]["enrichment"]["mode"] == "offline_default" assert payload["metadata"]["enrichment"]["pypi_enabled"] is False assert payload["metadata"]["enrichment"]["network_access_performed"] is False + assert payload["evidence_confidence"] == "local_manifest_only" + assert payload["summary"]["evidence_confidence"] == "local_manifest_only" + assert payload["metadata"]["evidence_confidence"] == "local_manifest_only" assert payload["trust_signal_notes"] == [ "PyPI components are present, but provenance enrichment was not enabled for this run." ] @@ -128,6 +131,7 @@ def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 assert payload["metadata"]["enrichment"]["mode"] == "opt_in_pypi" assert payload["metadata"]["enrichment"]["pypi_enabled"] is True assert payload["metadata"]["enrichment"]["pypi_timeout_seconds"] == 2.5 + assert payload["evidence_confidence"] == "enrichment_mocked" assert payload["notes"][1] == "PyPI provenance enrichment was requested explicitly." assert payload["trust_signal_notes"] == [] @@ -196,4 +200,5 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: assert payload["metadata"]["enrichment"]["mode"] == "opt_in_scorecard" assert payload["metadata"]["enrichment"]["scorecard_enabled"] is True assert payload["metadata"]["enrichment"]["scorecard_timeout_seconds"] == 4.25 + assert payload["evidence_confidence"] == "enrichment_mocked" assert payload["notes"][1] == "OpenSSF Scorecard enrichment was requested explicitly." diff --git a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py index 900aa17..8f23369 100644 --- a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py +++ b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py @@ -39,6 +39,7 @@ def test_cli_summary_json_writes_summary_only_file(tmp_path: Path) -> None: "suspicious_source": 0, "not_evaluated": 2, }, + "evidence_confidence": "sbom_present", } assert "unchanged" not in payload assert summary_path.read_text(encoding="utf-8").endswith("\n") @@ -97,6 +98,7 @@ def test_cli_summary_json_includes_policy_summary_when_policy_is_used(tmp_path: "warning": 1, "suppressed": 0, } + assert payload["evidence_confidence"] == "policy_matched" assert "enrichment" not in payload @@ -159,6 +161,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: }, }, } + assert payload["evidence_confidence"] == "enrichment_mocked" assert "policy" not in payload @@ -221,6 +224,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata: }, }, } + assert payload["evidence_confidence"] == "enrichment_mocked" assert "policy" not in payload diff --git a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py new file mode 100644 index 0000000..4ab8532 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from sbom_diff_risk.evidence_confidence import evidence_confidence_for_report +from sbom_diff_risk.models import ( + CompareReport, + Component, + EvidenceConfidence, + ReportComponents, + ReportEnrichmentMetadata, + ReportMetadata, + ReportSummary, +) +from sbom_diff_risk.policy_models import PolicyEvaluation, PolicyLevel, PolicyViolation + + +def test_evidence_confidence_defaults_to_local_manifest_only() -> None: + report = _minimal_report(before_format="requirements-txt", after_format="requirements-txt") + + assert evidence_confidence_for_report(report) is EvidenceConfidence.LOCAL_MANIFEST_ONLY + + +def test_evidence_confidence_marks_sbom_present() -> None: + report = _minimal_report(before_format="cyclonedx-json", after_format="cyclonedx-json") + + assert evidence_confidence_for_report(report) is EvidenceConfidence.SBOM_PRESENT + + +def test_evidence_confidence_marks_policy_matched() -> None: + report = _minimal_report( + before_format="requirements-txt", + after_format="requirements-txt", + policy_evaluation=PolicyEvaluation( + applied=True, + warning_violations=[ + PolicyViolation( + rule_id="new_package", + level=PolicyLevel.WARN, + message="New package matched local policy.", + ) + ], + ), + ) + assert evidence_confidence_for_report(report) is EvidenceConfidence.POLICY_MATCHED + + +def test_evidence_confidence_marks_mocked_enrichment_without_network() -> None: + report = _minimal_report( + before_format="requirements-txt", + after_format="requirements-txt", + enrichment=ReportEnrichmentMetadata( + mode="opt_in_pypi", + pypi_enabled=True, + pypi_network_access_performed=False, + network_access_performed=False, + ), + ) + + assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED + + +def test_evidence_confidence_marks_live_enrichment_when_network_access_was_performed() -> None: + report = _minimal_report( + before_format="requirements-txt", + after_format="requirements-txt", + enrichment=ReportEnrichmentMetadata( + mode="opt_in_pypi", + pypi_enabled=True, + pypi_network_access_performed=True, + network_access_performed=True, + ), + ) + + assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_LIVE + + +def test_evidence_confidence_allows_explicit_mock_override_for_constructed_snapshots() -> None: + report = _minimal_report( + before_format="requirements-txt", + after_format="requirements-txt", + evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, + enrichment=ReportEnrichmentMetadata( + mode="opt_in_pypi", + pypi_enabled=True, + pypi_network_access_performed=True, + network_access_performed=True, + ), + ) + + assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED + + +def _minimal_report( + *, + before_format: str, + after_format: str, + policy_evaluation: PolicyEvaluation | None = None, + evidence_confidence: EvidenceConfidence | None = None, + enrichment: ReportEnrichmentMetadata | None = None, +) -> CompareReport: + return CompareReport( + summary=ReportSummary(added=0, removed=0, changed=0, risk_counts={}), + components=ReportComponents( + added=[Component(name="requests", version="2.32.0", ecosystem="pypi")], + removed=[], + changed=[], + ), + risks=[], + metadata=ReportMetadata( + before_format=before_format, + after_format=after_format, + policy_evaluation=policy_evaluation, + evidence_confidence=evidence_confidence, + enrichment=enrichment or ReportEnrichmentMetadata(), + ), + ) diff --git a/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py b/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py index e24aaed..2ef060c 100644 --- a/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py +++ b/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py @@ -8,6 +8,7 @@ CompareReport, Component, ComponentChange, + EvidenceConfidence, ProvenanceEvidence, ProvenanceFileEvidence, ProvenanceStatus, @@ -58,6 +59,7 @@ def test_provenance_report_json_includes_provenance_policy_summary() -> None: "warning": 1, "suppressed": 0, } + assert payload["summary"]["evidence_confidence"] == "enrichment_mocked" assert payload["summary"]["enrichment"] == { "status": "used", "mode": "opt_in_pypi", @@ -403,6 +405,7 @@ def _build_sample_provenance_report() -> tuple[CompareReport, Path, Path]: strict=False, stub=False, policy_evaluation=policy_evaluation, + evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, enrichment=ReportEnrichmentMetadata( mode="opt_in_pypi", pypi_enabled=True, diff --git a/tools/sbom-diff-and-risk/tests/test_reports.py b/tools/sbom-diff-and-risk/tests/test_reports.py index eeeec47..c076f27 100644 --- a/tools/sbom-diff-and-risk/tests/test_reports.py +++ b/tools/sbom-diff-and-risk/tests/test_reports.py @@ -7,6 +7,7 @@ from sbom_diff_risk.models import ( CompareReport, Component, + EvidenceConfidence, ProvenanceEvidence, ProvenanceFileEvidence, ProvenanceStatus, @@ -124,6 +125,7 @@ def test_report_json_keeps_legacy_sections() -> None: assert set(payload) >= { "summary", + "evidence_confidence", "components", "risks", "policy_evaluation", @@ -141,6 +143,8 @@ def test_report_json_keeps_legacy_sections() -> None: } assert payload["metadata"]["policy_evaluation"] == payload["policy_evaluation"] assert payload["metadata"]["enrichment"] == payload["enrichment_metadata"] + assert payload["metadata"]["evidence_confidence"] == payload["evidence_confidence"] + assert payload["summary"]["evidence_confidence"] == payload["evidence_confidence"] assert "provenance_policy" not in payload assert "provenance_policy_impact" not in payload @@ -166,6 +170,7 @@ def test_report_json_offline_enrichment_metadata_is_stable_by_default() -> None: "suspicious_source": 0, "not_evaluated": 2, }, + "evidence_confidence": "sbom_present", } assert payload["metadata"]["enrichment"] == { "mode": "offline_default", @@ -206,6 +211,7 @@ def test_report_json_summary_includes_policy_status_when_policy_is_used() -> Non "warning": 1, "suppressed": 0, } + assert payload["summary"]["evidence_confidence"] == "policy_matched" assert "enrichment" not in payload["summary"] @@ -318,6 +324,7 @@ def test_reports_include_provenance_policy_details_for_v2_policy() -> None: strict=False, stub=False, policy_evaluation=policy_evaluation, + evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, ), notes=["PyPI provenance enrichment was requested explicitly."], ) diff --git a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py index 948604a..3bcbf45 100644 --- a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py +++ b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py @@ -8,6 +8,7 @@ CompareReport, Component, ComponentChange, + EvidenceConfidence, ReportComponents, ReportEnrichmentMetadata, ReportMetadata, @@ -47,6 +48,7 @@ def test_scorecard_report_json_summary_includes_enrichment_status() -> None: "warning": 1, "suppressed": 0, } + assert payload["summary"]["evidence_confidence"] == "enrichment_mocked" assert payload["summary"]["enrichment"] == { "status": "used", "mode": "opt_in_scorecard", @@ -236,6 +238,7 @@ def _build_sample_scorecard_report() -> tuple[CompareReport, Path, Path]: strict=False, stub=False, policy_evaluation=policy_evaluation, + evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED, enrichment=ReportEnrichmentMetadata( mode="opt_in_scorecard", scorecard_enabled=True,