diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3e193fc..4dd473b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: - / - /benchmark - /fuzz + - /repository-process-spawn - /xtask schedule: interval: weekly diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e1fd53..ede3834 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,9 @@ jobs: with: persist-credentials: false + - name: Install pinned toolchain + run: rustup show + - name: Install pinned Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -95,14 +98,11 @@ jobs: "$documentation_tools/bin" \ "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" - - name: Verify documentation integrity laws - run: python3 -m unittest discover -s scripts -p 'test_*.py' -v + - name: Verify malformed-input refusal laws + run: cargo xtask documentation-refusal-check - - name: Check Markdown and internal links - run: python3 scripts/check_markdown.py - - - name: Check workflow syntax - run: python3 scripts/check_workflows.py + - name: Check documentation and workflows + run: cargo xtask documentation-integrity-check - name: Check repository whitespace run: git diff --check "$(git hash-object -t tree /dev/null)" HEAD @@ -134,6 +134,9 @@ jobs: - name: Prepare deterministic fuzz seeds run: cargo xtask prepare-fuzz-corpus + - name: Build every fuzz target + run: cargo xtask fuzz build --profile smoke + - name: Exercise every fuzz target run: cargo xtask fuzz run --profile smoke diff --git a/.gitignore b/.gitignore index 21b47e8..f34925c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,8 @@ /scripts/documentation-tools/node_modules/ **/*.rs.bk -# Python checker bytecode -__pycache__/ -*.py[cod] +# Agent workspace state +.graft/ # macOS .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 192c48f..e9029eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,87 @@ after its public API and format compatibility policies are established. ### Changed +- Documentation corpus selection, pinned tool admission, Markdown and fragment + checks, workflow linting, Dependabot coverage, and Node lock-graph policy now + run through bounded Rust `xtask` code; CI and `cargo xtask verify` use that + boundary, and the seven superseded Python checkers have been removed. The + boundary rejects duplicate repository JSON fields and unlocked installer + substitutions, admits only the exact reviewed Node lock artifact, retains + simultaneous Markdown and link failures, retains both the primary tool + failure and a simultaneous snapshot-cleanup failure, parses documentation + workflow commands as YAML, rejects guarded or non-string `run` values, + preserves declarations after Dependabot directory lists, refuses duplicate + Dependabot YAML mapping keys before semantic admission, compares Dependabot + maintenance fields as typed YAML values, requires every reviewed + documentation CI command and the pinned Node setup action exactly once, + admits only the exact pinned checkout and Node setup actions, requires actions + and commands to execute in one reviewed order, rejects checkout overrides and + unreviewed action steps, refuses + alternate setup-node actions, requires the reviewed Node version, rejects + unreviewed workflow/job run defaults and step execution fields, pins the + documentation runner and job deadline, requires the exact top-level + `contents: read` permission mapping, rejects guarded or failure-tolerant + documentation jobs and required steps, refuses step mappings that define + neither a reviewed action nor a run command, requires each Dependabot update + block to choose exactly one directory field form, and requires the + documentation workflow to run for pushes to `main` and every pull request, + executes malformed Markdown and workflow evidence through the named + `cargo xtask documentation-refusal-check` boundary instead of a + zero-match-successful libtest substring filter, + bounds each admitted documentation source to 4 MiB and each selected corpus + to 64 MiB before external tools start, retains every selected source + identity, refuses device, inode, size, modification-time, or change-time + drift before and after each external tool, re-inventories complete corpus + membership so newly added sources cannot bypass those tools, revalidates + retained source identities after each rebuilt membership set so same-path + replacement cannot cross the inventory boundary, executes the tools against + a private snapshot copied from the admitted source descriptors so transient + path substitution cannot redirect their reads, copies bounded regular + non-Markdown namespace files exactly so link-fragment checks observe admitted + target bytes, refuses nonregular namespace targets instead of substituting + placeholders, retains each fixed policy file identity through semantic + admission and external tool execution so a replaced path cannot validate + bytes from a superseded file, + and applies a two-minute deadline across Git inventory, validation-tool + execution, and output collection. Validation tools clear the inherited + environment and admit only the executable search path and `C` locale, so + preload hooks and host-specific configuration cannot alter evidence. + Git inventory uses a separate explicit profile that also nulls system and + global configuration and disables optional locking, so repository overrides + cannot redirect selection or cause incidental index writes. Failed Git + inventory commands report their exit status and diagnostic before attempting + path-stream decoding, so malformed stdout cannot mask the authoritative + failure. + Git-backed process fixtures clear the inherited environment, explicitly + admit the executable search path and `C` locale, ignore system and global Git + configuration, and preserve non-UTF-8 template paths without lossy + conversion. Repository-backed Git fixtures share one bounded process + authority with dedicated groups, null standard streams, and a two-minute + deadline. Documentation Git inventory and tools start from one retained + repository directory handle, so transient replacement of the ambient + repository path cannot redirect validation. Retained and per-spawn directory + descriptors are allocated at descriptor 3 or above, so child standard-stream + setup cannot overwrite the working-directory authority. + Source-structure inspection refuses nonregular executable candidates instead + of silently omitting them from the Python and hard-line-limit policy. + Terminal signals now become typed refusals while an external repository task + is active, so captured and inherited child groups are killed and reaped + before `xtask` returns. Captured-output readers finish while the process-group + leader remains waitable, and no cleanup path can address its numeric group + identity after the child has been reaped. Descendant cleanup evidence now + uses a pre-established socket disconnect instead of elapsed-time reachability + polling. +- Fuzz build and run plans now carry external process deadlines from the + reviewed campaign policy. Both smoke and scheduled CI campaigns build every + target under the separate build deadline before applying per-target run + deadlines. Workflow contract evidence parses only executable `run` scalars in + the reviewed fuzz jobs, so comments, names, and environment values cannot + impersonate required build or run commands. Run deadlines use checked + addition of the exploration budget and process-grace interval before + process-group execution. +- The fuzz dependency-policy gate now grants exact MIT license exceptions to + the reviewed `memchr` 2.8.3 and `zmij` 1.0.23 transitive dependencies while + retaining Apache-2.0 as the default license allowlist. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, @@ -30,11 +111,33 @@ after its public API and format compatibility policies are established. - Golden File Worldline verification now runs through a dependency-isolated Rust `xtask`, cross-checks every identity-bearing digest against external `b3sum`, and CI refuses Rust, Python, or shell source modules that exceed the - documented 500-physical-line hard maximum, including test modules. + documented 500-physical-line hard maximum, including test modules and + executable sources regardless of filename suffix. - Repository source verification now uses capability-relative, no-follow file - opens and verifies repository-root identity after Git inventory, so a - persistent root replacement or source path replaced with a symlink is - refused before source bytes are read. + opens, normalizes symlink refusal at that capability boundary across Unix + error conventions, starts Git inventory through a descriptor duplicated from + that same admitted root, and verifies repository-root identity before + inventory, after inventory, and after source scanning. Persistent or + transient ambient-root substitution therefore cannot split path selection + from source reads, and a source path replaced with a symlink is refused. The + pure Rust boundary also refuses + `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every + executable regular file regardless of filename suffix, including raw + non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment + shebangs parse exact and unambiguous abbreviated long options, combined + short-option clusters, assignments, quoting, and split strings before + classifying only the selected utility, so later command arguments cannot + impersonate Python and unresolved utility substitutions fail closed. Tracked + file modes are admitted from the Git index and must agree with the worktree, + so a staged executable cannot defer Python screening until the next checkout. + Source execution, shebang, and physical-line evidence now come from one + admitted file descriptor whose identity is revalidated after each read phase, + so path replacement or in-place mutation cannot splice different file states + into one verification result. +- Git path inventory failures now remain primary when child cleanup, waiting, + or diagnostic collection also fails; the secondary failure remains typed and + inspectable. Empty path records and unterminated path bytes produce distinct, + accurate typed diagnostics. - The repository `cargo xtask` alias and Rust command contract are now explicitly silent on success and emit one typed `Error:` diagnostic with exit status 1 on refusal; untrusted control characters are escaped so the @@ -42,6 +145,9 @@ after its public API and format compatibility policies are established. - Golden protocol framing, field, hexadecimal, path, mutation-operation, and fixed-width value decoders now share a bounded fuzz surface backed by precise table-driven malformed-corpus refusals. +- Duplicate-refusing repository JSON admission now has a one-mebibyte fuzz + boundary with deterministic evidence for valid nested input, malformed JSON, + excessive nesting, and duplicate members at nested object depth. - Deterministic fuzz seed materialization now uses a capability-bound Rust `xtask`, syncs and atomically publishes derived seed files without mutating hard-link targets, recovers interrupted fixed-name staging files, cleans diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bfea61..2e6db04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,18 +38,18 @@ npm ci --prefix scripts/documentation-tools --ignore-scripts --no-audit --no-fun cargo install lychee --version 0.21.0 --locked go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 export PATH="$PWD/scripts/documentation-tools/node_modules/.bin:$PATH" -python3 scripts/check_markdown.py -python3 scripts/check_workflows.py +cargo xtask documentation-integrity-check git diff --check git diff --cached --check ``` -The checker admits tracked Markdown plus nonignored new Markdown and refuses -any other tool version. Build products, generated Rustdoc, fuzz artifacts, -and other ignored files therefore cannot change the result. Link validation -checks local files and fragments with network access disabled; external-site -availability cannot change the result. The two Git commands check unstaged -and staged whitespace errors separately. +The Rust checker admits tracked Markdown and workflows plus nonignored new +files, verifies the committed Node lock graph and Dependabot coverage, and +refuses any other tool version. Build products, generated Rustdoc, fuzz +artifacts, and other ignored files therefore cannot change the result. Link +validation checks local files and fragments with network access disabled; +external-site availability cannot change the result. The two Git commands +check unstaged and staged whitespace errors separately. ## Development checks diff --git a/Cargo.lock b/Cargo.lock index adda953..5775a7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.9" @@ -216,7 +222,7 @@ checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -235,6 +241,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "fs-set-times" version = "0.20.3" @@ -246,6 +258,24 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown", +] + [[package]] name = "hybrid-array" version = "0.4.13" @@ -283,6 +313,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "keep" version = "0.0.0" @@ -329,6 +365,12 @@ dependencies = [ "digest", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "once_cell" version = "1.21.4" @@ -359,6 +401,13 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" +[[package]] +name = "repository-process-spawn" +version = "0.0.0" +dependencies = [ + "rustix", +] + [[package]] name = "rustix" version = "1.1.4" @@ -382,12 +431,74 @@ dependencies = [ "rustix", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "syn" version = "2.0.119" @@ -399,6 +510,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "typenum" version = "1.20.1" @@ -613,4 +735,26 @@ dependencies = [ "cap-fs-ext", "cap-std", "md-5", + "repository-process-spawn", + "rustix", + "serde", + "serde_json", + "signal-hook", + "yaml-rust2", +] + +[[package]] +name = "yaml-rust2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" +dependencies = [ + "arraydeque", + "hashlink", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 397b698..7049a72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,14 +24,14 @@ name = "streaming_cdc" harness = false [workspace] -members = [".", "benchmark", "xtask"] +members = [".", "benchmark", "repository-process-spawn", "xtask"] resolver = "3" [lints] workspace = true [workspace.lints.rust] -unsafe_code = "forbid" +unsafe_code = "deny" missing_docs = "deny" unused_must_use = "deny" unreachable_pub = "deny" diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index cb1a37a..da5cdc8 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -52,6 +52,7 @@ Keep keeps its durable truth in a small set of known places. | `docs/recovery/` | The crash-point catalog, the publication protocol, and the documented lawful state recovery must reach. | | `docs/adr/` | Slugged ADRs for decisions that cut across subsystems or predate a colocated home. See §3.10. | | `docs///rationale.md` | Colocated decision record for a decision scoped to one concept: the decision, alternatives rejected, and why. See §3.10. | +| `docs/dependencies/` | Dependency and development-tool admission records: scope, selected features, resolved graph, risks, and review triggers. | | `CONTRIBUTING.md` | Contributor-facing operational contract: what to read, required local checks, and PR expectations. | | `CHANGELOG.md` | Release-visible historical ledger. | | `SECURITY.md` | Vulnerability reporting and support posture. | @@ -533,35 +534,40 @@ Documentation quality requires both deterministic checks and human judgment. Run for documentation changes: ```bash -python3 scripts/check_markdown.py +cargo xtask documentation-integrity-check git diff --check git diff --cached --check ``` -Use `markdownlint-cli2` 0.23.2. The repository-owned configuration defines -the default Markdown input set, imports `.gitignore`, and records deliberate -rule choices. The checker admits tracked Markdown plus nonignored new -Markdown, disables config globs for that invocation, and refuses a different -tool version. It also runs `lychee` 0.21.0 offline with fragment checking, so -external-site availability cannot affect the result. Run it from the -repository root. The two Git commands inspect unstaged and staged whitespace -errors separately. - -When workflows change, also run: - -```bash -python3 scripts/check_workflows.py -``` - -The workflow checker requires `actionlint` 1.7.12 and refuses another version. +Use `markdownlint-cli2` 0.23.2. The repository-owned configuration records +deliberate rule choices. The Rust checker selects tracked Markdown plus +nonignored new Markdown, disables configuration globs for that invocation, +and refuses a different tool version. It also runs `lychee` 0.21.0 offline +with fragment checking, so external-site availability cannot affect the +result. Before external tools start, corpus admission refuses any source over +4 MiB and any selected corpus over 64 MiB. Run it from the repository root. +The two Git commands inspect +unstaged and staged whitespace errors separately. + +The same Rust command checks workflows with `actionlint` 1.7.12 and refuses +another version. It also verifies the committed Node lock graph, Dependabot +manifest coverage, and the documentation job's delegation to this command. The dedicated `documentation` job in `.github/workflows/ci.yml` installs the -pinned tools, runs these repository-owned checks, and verifies repository -whitespace before admitting the result as CI evidence. +pinned tools, runs `cargo xtask documentation-refusal-check`, runs the +repository-owned integrity command, and verifies repository whitespace before +admitting the result as CI evidence. The named refusal command constructs and +executes both malformed-input scenarios directly, so removing or renaming +either scenario breaks the command instead of producing a successful +zero-test result. The Rust workflow contract pins that job to `ubuntu-latest` +with a ten-minute deadline, rejects workflow and job run defaults, and admits +only the reviewed action and command step fields. A custom shell, working +directory, environment, or other unreviewed execution modifier cannot +impersonate a required command. CI SHOULD block on facts it can determine reliably: - malformed Markdown; -- broken internal links and anchors, once link checking is available; +- broken internal links and anchors; - failed doctests declared runnable (`cargo test --workspace --doc --locked`); - stale generated reference — public API documentation that no longer diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 6f6775e..0905782 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -142,7 +142,7 @@ All crates MUST inherit workspace lints. # Cargo.toml [workspace.lints.rust] -unsafe_code = "forbid" +unsafe_code = "deny" missing_docs = "deny" unused_must_use = "deny" unreachable_pub = "deny" @@ -204,7 +204,13 @@ Each crate: #![warn(clippy::cargo)] ``` -Clippy’s `pedantic` group is explicitly aggressive and can produce false positives; that is acceptable here. Exceptions must be local and justified rather than weakening the workspace globally. (⁠[Rust Docs](https://doc.rust-lang.org/stable/clippy/lints.html?utm_source=chatgpt.com)) +Ordinary crates strengthen the workspace's `unsafe_code = "deny"` to +`forbid`. A dedicated unsafe-boundary crate MAY carry a crate-level, +reason-bearing allowance only after satisfying §4.2. + +Clippy’s `pedantic` group is explicitly aggressive and can produce false +positives; that is acceptable here. Exceptions must be local and justified +rather than weakening the workspace globally. ### **3.2 No broad lint suppression** @@ -271,7 +277,8 @@ cargo clippy \ ### **4.1 Default rule** -`unsafe` is forbidden throughout Keep V1. +`unsafe` is forbidden throughout Keep V1 except inside a dedicated crate +admitted under §4.2. ```rust #![forbid(unsafe_code)] @@ -288,9 +295,10 @@ Do not permit unsafe merely because storage engines often eventually use: Earn it later. -### **4.2 Future unsafe admission** +### **4.2 Unsafe admission** -If unsafe becomes demonstrably necessary, it MUST live in a dedicated crate such as: +If unsafe becomes demonstrably necessary, it MUST live in a dedicated crate +such as: ```text keep-platform @@ -306,12 +314,20 @@ That crate MUST contain: - property tests around the safe wrapper; - platform-specific integration tests; - a written alternative analysis; -- benchmarks proving the unsafe implementation earns its existence. +- measurements proving that safe alternatives cannot establish the required + behavior; +- benchmarks when performance is part of the justification. Every unsafe block MUST have a nearby `SAFETY:` explanation proving all preconditions. “Required for performance” is not a proof. +The only current admission is +[`repository-process-spawn`](../repository-process-spawn/src/lib.rs), governed +by the +[descriptor-bound child working-directory decision](adr/0006-descriptor-bound-child-working-directory.md). +Its one hook calls only POSIX async-signal-safe `fchdir` between fork and exec. + --- ## **5. Repository and Crate Structure** @@ -461,7 +477,18 @@ Hard CI limits: Count physical lines for deterministic enforcement. Blank and comment lines remain part of the maintainability surface; reviewers should also examine -logical structure. +logical structure. The hard maximum applies to every executable source file +regardless of its filename suffix. Source classification, executable-shebang +inspection, and line counting MUST use one admitted file descriptor; reopening +the pathname between checks would permit replacement to splice evidence from +different files. The scanner MUST revalidate both the descriptor identity and +the current repository path after each read phase, and refuse concurrent +replacement or in-place mutation. + +Source Git inventory and source-file admission MUST descend from the same +opened repository capability. Inventory MUST start through a descriptor-bound +child working directory; an ambient pathname passed to +`Command::current_dir` cannot establish the required authority continuity. A file above 300 lines MUST begin with a decomposition issue or contain an approved exception explaining why splitting it would damage locality. @@ -2041,6 +2068,8 @@ Also forbidden: - hidden filesystem access; - hidden network access; - hidden allocation proportional to input; +- Python source files, including `.py`, `.pyw`, and executable Python shebangs + in regular files regardless of filename suffix; - random IDs where content identity is required; - wall-clock time in deterministic algorithms; - hashing arbitrary serializer output; diff --git a/docs/adr/0006-descriptor-bound-child-working-directory.md b/docs/adr/0006-descriptor-bound-child-working-directory.md new file mode 100644 index 0000000..429761a --- /dev/null +++ b/docs/adr/0006-descriptor-bound-child-working-directory.md @@ -0,0 +1,91 @@ +# ADR-0006: Descriptor-Bound Child Working Directory + +- Status: Accepted +- Date: 2026-07-28 +- Owners: Keep repository verification +- Related issue: none — review remediation for pull request 61 +- Depends on: ADR-0004 + +## Context + +Repository verification inventories Git paths and runs pinned validation +tools. Opening the repository as a capability protects file reads, but passing +its ambient pathname to a child creates another replacement window. An +attacker could move the admitted directory, substitute another repository +while source or documentation inventory runs, and restore the original before +the final identity check. + +Retaining the directory alone does not bind a selected corpus path to the file +admitted at that path. A source can be renamed, replaced while a tool reads it, +and restored without changing the retained repository directory identity. + +The standard library accepts only a pathname for +`std::process::Command::current_dir`. On macOS, an open directory exposed +through `/dev/fd` cannot be used as that pathname. Changing the parent process +directory through the safe Rustix `fchdir` wrapper was measured and rejected: +parallel xtask tests observed the temporary global state and failed. + +The remaining standard-library hook is +`std::os::unix::process::CommandExt::pre_exec`, which is unsafe because code +between fork and exec must obey strict rules. + +## Decision + +Keep isolates the hook in the private `repository-process-spawn` workspace +crate. The crate admits one operation: + +1. Own a close-on-exec duplicate of the admitted repository directory, + allocated at descriptor 3 or above so child standard-stream setup cannot + overwrite it. +2. Register a child setup hook that calls only Rustix `fchdir`. +3. Spawn through the existing bounded process adapters. +4. Retain every selected source's device, inode, size, modification time, and + change time. +5. Reopen every selected path through the repository capability and compare + the admitted identity with the current path before and after every tool. + +POSIX specifies `fchdir` as async-signal-safe. The hook performs no allocation, +locking, buffered I/O, ambient path lookup, or user callback. The descriptor +closes on successful exec. A setup failure is returned by `Command::spawn`. +Both the retained process-directory handle and the per-spawn duplicate use the +same non-standard-descriptor minimum. + +The workspace denies unsafe code by default. Only the dedicated crate carries +an explained `unsafe_code` allowance. It contains no storage, identity, format, +network, or application policy. + +## Alternatives considered + +- Rechecking the repository pathname before and after tool execution does not + detect a transient substitution. +- Reopening selected source paths without retaining and comparing the admitted + identity cannot distinguish the selected file from a later replacement. +- `/dev/fd/` and `/proc/self/fd/` are not a portable child working + directory. The former is not traversable as a directory on macOS, and the + latter is not available there. +- Changing the parent directory under a mutex still mutates process-global + state visible to threads outside that mutex. +- Copying the repository into a temporary tree changes Git, ignore, link, and + untracked-file semantics and creates an unbounded materialization. +- Reimplementing subprocess management would duplicate the standard library's + file-descriptor, environment, signal, and error handling. + +## Consequences + +Source inventory, documentation inventory, and documentation tools start in +the exact opened repository even if its pathname is replaced. Parent process +state remains unchanged, so parallel tests and readers are deterministic. + +External tool output is admitted only while every selected path still has the +admitted device, inode, size, modification time, and change time +unchanged. Persistent replacement, in-place mutation, and +substitute-then-restore changes produce a typed corpus refusal. + +The boundary is Unix-specific and deliberately narrow. Any additional unsafe +operation, child hook, captured state, or consumer requires a new decision and +new executable evidence. The crate's regression test replaces the ambient +path, proves the child reads the retained directory, and proves the parent +working directory is unchanged. + +No durable or authoritative state is written. Failure refuses the repository +task, so no recovery protocol is required. diff --git a/docs/adr/0007-terminal-signal-process-group-guard.md b/docs/adr/0007-terminal-signal-process-group-guard.md new file mode 100644 index 0000000..1b684ad --- /dev/null +++ b/docs/adr/0007-terminal-signal-process-group-guard.md @@ -0,0 +1,89 @@ +# ADR-0007: Terminal Signal Process-Group Guard + +- Status: Accepted +- Date: 2026-07-28 +- Owners: Keep repository verification +- Related issue: none — review remediation for pull request 61 +- Depends on: ADR-0006 + +## Context + +Repository tasks run external tools in dedicated process groups. A deadline or +collection failure terminates the whole group, but the operating system's +default terminal-signal action terminates the `xtask` parent immediately. +Signals such as `SIGINT` therefore bypass Rust cleanup and can leave an +isolated child or descendant running. + +Captured output creates a second wait boundary. A child leader can exit while +a descendant retains an inherited pipe, so guarding only the child wait does +not cover the complete operation. + +Git path inventory has the same boundary. A damaged repository can stall Git, +and a descendant can retain either output pipe after Git exits. + +## Decision + +While an external repository task is active, `xtask` installs one process-wide +guard for `SIGINT`, `SIGTERM`, `SIGHUP`, and `SIGQUIT`. A dedicated signal +thread records the first signal for every active operation. Child waits and +captured-output readers poll that state at the existing ten-millisecond process +interval. + +Git path inventory and documentation validators use the same two-minute +deadline, process-group isolation, signal guard, and reader cleanup. Git keeps +its 16 MiB path-stream and 64 KiB diagnostic bounds through explicit +per-stream capture limits. + +An observed terminal signal becomes a typed `ProcessError::Interrupted` +refusal. The normal failure path then sends `SIGKILL` to the dedicated child +process group, kills and reaps the child, and joins captured-output readers +before returning. Captured processes collect and join both readers before the +successful child wait consumes the process-group leader's waitable identity. +Every reader failure that can invoke group cleanup therefore occurs while that +identity is still owned, and no cleanup path addresses a numeric process-group +ID after the child has been reaped. Output-limit and late-interrupt refusals +found after a successful wait require no cleanup because the child and readers +have already terminated. + +The terminal signal is not sent directly to the child group; one cleanup +authority avoids races between signal delivery and mandatory process-group +termination. + +When no external operation is active, the guard restores the signal's default +behavior. A signal observed while the final operation retires, or a second +signal received during cleanup, also restores default termination instead of +being swallowed. Handler registration is initialized once because removing a +`signal-hook` action does not restore a previous operating-system handler. The +[signal-hook dependency admission](../dependencies/signal-hook-0.4.4.md) +records the selected package boundary. + +## Alternatives considered + +- Relying on the default terminal action abandons Rust cleanup and can strand + descendants. +- Forwarding the original terminal signal directly to the child group races + mandatory cleanup. On macOS, a descendant can exit from `SIGINT` before the + subsequent group kill, which makes the second operation report `EPERM` + despite successful termination. +- A parent-death signal is not portable across the supported Unix platforms + and does not cover descendants that change their parent relationship. +- Polling only the child status misses descendants that retain captured output + pipes after the child leader exits. + +## Consequences + +Terminal interruption follows the same typed cleanup and reaping protocol as +timeouts and collection failures. Unbounded child waits and output collection +now wake at most one process-poll interval after the signal thread records an +interrupt. + +Signal registration is process-global, but active state is operation-local and +supports concurrent repository tasks. No lock is held while a child is +spawned, polled, killed, reaped, or read. + +The guard is private to `xtask`. It changes no Keep library API, content +identity, durable format, or recovery protocol. Regression evidence sends +`SIGINT` to a supervisor with an isolated descendant, requires the exact typed +refusal, and proves the descendant is no longer reachable. The test establishes +a witness socket before cleanup and waits for the kernel-reported disconnect; +elapsed time and scheduler latency do not classify descendant termination. diff --git a/docs/adr/README.md b/docs/adr/README.md index e2553f1..0521dfb 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -57,3 +57,5 @@ encryption, concurrency, or public-API surface it governs. - [ADR-0003: Deterministic content-defined chunking profiles](0003-deterministic-content-defined-chunking-profiles.md) - [ADR-0004: Hexagonal boundary architecture](0004-hexagonal-boundary-architecture.md) - [ADR-0005: Durable segment store protocol](0005-durable-segment-store-protocol.md) +- [ADR-0006: Descriptor-bound child working directory](0006-descriptor-bound-child-working-directory.md) +- [ADR-0007: Terminal signal process-group guard](0007-terminal-signal-process-group-guard.md) diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 429d34c..ffc681b 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -1,4 +1,4 @@ -# Dependency Admission: cap-std and cap-fs-ext 4.0.2 +# Dependency Admission: cap-std, cap-fs-ext 4.0.2, and rustix 1.1.4 - Status: Accepted for repository-task filesystem boundaries only - Date: 2026-07-26 @@ -8,8 +8,9 @@ ## Admitted use -Keep admits the exactly pinned `cap-std` 4.0.2 and `cap-fs-ext` 4.0.2 packages -only behind the `xtask` crate's `repository-tasks` feature. +Keep admits the exactly pinned `cap-std` 4.0.2, `cap-fs-ext` 4.0.2, and +`rustix` 1.1.4 packages only behind the `xtask` crate's `repository-tasks` +feature. `cap-std::fs::Dir` pins the admitted repository or corpus directory and opens entries relative to that capability. `cap-fs-ext` supplies no-follow and @@ -24,6 +25,25 @@ These packages are absent from Keep's published library graph, public API, content identities, durable formats, and production behavior. No dependency-owned type crosses out of the private repository-task adapter. +The bounded subprocess adapter uses Rustix's safe process API to send +`SIGKILL` to a dedicated child process group after a subprocess deadline or +collection failure. This prevents descendants that inherited an output pipe +from surviving the failed repository task. +The +[signal-hook dependency admission](signal-hook-0.4.4.md) +records the terminal-signal guard that routes interruption through the same +authoritative cleanup boundary. + +Documentation verification also duplicates the admitted repository directory +handle and uses the isolated `repository-process-spawn` crate to start Git and +validation tools from that exact directory. Its child-only setup hook performs +one POSIX async-signal-safe `fchdir` after fork and before exec. Parent process +state never changes. A transient ambient-path replacement therefore cannot +redirect corpus inventory or tool execution. +The +[descriptor-bound child working-directory decision](../adr/0006-descriptor-bound-child-working-directory.md) +records the unsafe-boundary invariants and rejected alternatives. + ## Why the standard library is insufficient Checking a path and then reopening it with `std::fs` leaves a @@ -34,15 +54,16 @@ the admitted object. The standard library also has no portable API that combines capability-relative opens with no-follow and nonblocking semantics. Recreating that boundary locally would require operating-system-specific flags, handle -conversion, and path-resolution code. That would exceed a small local helper, -duplicate security-sensitive upstream work, and require unsafe code that Keep -otherwise forbids. +conversion, descriptor-relative directory changes, and path-resolution code. +That would exceed a small local helper, duplicate security-sensitive upstream +work, and require unsafe code that Keep otherwise forbids. ## Features and resolved graph -Both direct dependencies disable default features. Keep enables only -`cap-fs-ext`'s `std` feature; `cap-std` has no enabled feature. Both declarations -are optional and are activated solely by `repository-tasks`. +All three direct dependencies disable default features. Keep enables only +`cap-fs-ext`'s `std` feature and Rustix's `process` and `std` features; +`cap-std` has no enabled feature. All declarations are optional and are +activated solely by `repository-tasks`. The locked non-Windows graph introduced for this boundary is: @@ -66,29 +87,32 @@ Windows resolution additionally retains the locked `windows-sys`, ## Safety, licensing, and compatibility -Both direct packages declare +The capability packages declare `Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT`; Keep selects an admitted -license through repository policy. Their manifests declare no Rust-version -floor. Compatibility is therefore established only by Keep's pinned stable, -MSRV, debug, release, Clippy, dependency-policy, and advisory lanes. - -The capability packages and their platform dependencies may contain unsafe -code around operating-system calls and handles. Keep-owned code invokes only -their safe APIs, retains handles in private adapter types, checks resulting -metadata, bounds reads, and never treats the dependency as proof of content -identity. `cargo deny` and RustSec checks remain mandatory point-in-time -evidence; they do not transfer Keep's unsafe-code guarantee to dependencies. +license through repository policy. Rustix declares `Apache-2.0 OR MIT`. +Their manifests declare no Rust-version floor. Compatibility is therefore +established only by Keep's pinned stable, MSRV, debug, release, Clippy, +dependency-policy, and advisory lanes. + +The admitted packages and their platform dependencies may contain unsafe code +around operating-system calls and handles. Keep-owned code invokes only their +safe APIs, retains handles and process identifiers in private adapter types, +checks resulting metadata, bounds reads, and never treats a dependency as +proof of content identity. `cargo deny` and RustSec checks remain mandatory +point-in-time evidence; they do not transfer Keep's unsafe-code guarantee to +dependencies. ## Failure and recovery boundaries -An open, metadata, or read failure is a typed refusal. The task never repairs, -rewrites, or substitutes repository data. Retained handles exist only for one -verification process and carry no durability or recovery semantics. +An open, metadata, read, descriptor-duplication, child-directory setup, or +child-spawn failure is a typed refusal. The task never repairs, rewrites, or +substitutes repository data. Retained handles exist only for one verification +process and carry no durability or recovery semantics. Keep can remove these dependencies without changing public or durable behavior by replacing them with an equally portable, safe implementation that preserves -capability-relative, no-follow, nonblocking, regular-file, and retained-handle -tests on every supported platform. +capability-relative, no-follow, nonblocking, regular-file, retained-handle, and +whole-process-group cleanup tests on every supported platform. Reopen this admission if either direct version, selected feature, resolved graph, license, supported platform, handle-retention invariant, or diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index e0991eb..a7ef879 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -31,18 +31,67 @@ before extraction: | `actionlint_1.7.12_linux_amd64.tar.gz` | `8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8` | | `lychee-x86_64-unknown-linux-gnu.tar.gz` | `a06547250f10021dcafc6ed5bb20fca75835b65711745b63cfdda34c29ff6a73` | -The repository checkers then verify each executable's reported version before -admitting its output as evidence. A missing tool, changed archive, unexpected -version, empty input corpus, or tool failure refuses the check. +The Rust `cargo xtask documentation-integrity-check` boundary verifies the +structure and exact BLAKE3 digest of the committed Node lock artifact and the +exact BLAKE3 digest of the reviewed installer. The byte-exact lock admission +refuses altered, omitted, or additional package records. The boundary parses +the CI workflow as YAML and admits only reviewed, unguarded `run` fields from +the documentation job, then verifies each executable's reported version +before admitting its output as evidence. A missing tool, changed archive, +unexpected version, empty input corpus, guarded or unreviewed command, or tool +failure refuses the check. ## Determinism and network posture -The Markdown and workflow checkers derive their inputs from Git's tracked and +The Rust task derives the Markdown and workflow inputs from Git's tracked and repository-nonignored source paths. Generated Rustdoc, build outputs, ignored fuzz artifacts, ignored vendor trees, and user-global ignore policy cannot -enter either input set. Missing tracked paths are treated as pending deletions; -Git-trackable nonregular paths such as symlinks are refused. Non-trackable -special files such as FIFOs cannot enter the Git-selected corpus. +enter either input set. Missing tracked paths are treated as pending +deletions; Git-trackable nonregular paths such as symlinks and tracked paths +replaced by FIFOs are refused. Non-trackable special files cannot enter the +Git-selected corpus. + +Each Git path inventory runs in a dedicated process group under a two-minute +deadline that covers the child and both output readers. Standard output retains +at most the 16 MiB path-stream bound, and diagnostics retain at most 64 KiB. A +timeout, terminal signal, reader failure, or exceeded bound terminates the +whole group and reaps the child before the task refuses. + +Inventory commands also clear the inherited environment. They admit only the +executable search path, the `C` locale, null system and global Git +configuration, and disabled optional locking. Repository directory, worktree, +index, object, and configuration overrides therefore cannot redirect the +selected path set, and read-only inventory cannot opportunistically refresh +the index. + +Documentation corpus tests construct their Git fixtures through the same +bounded process authority. Each fixture command runs in a dedicated process +group with a two-minute deadline and null output streams, so a stalled command +or descendant cannot outlive the test boundary. The helper clears the inherited +environment, then admits only the ambient executable search path, the `C` +locale, and explicit null system and global Git configuration. Ambient Git +directory, worktree, index, object, and configuration variables cannot redirect +a fixture. + +Git inventory and each validation tool start through one retained repository +directory handle. Child-only setup changes directory through that handle after +fork and before exec; the parent working directory does not change. Replacing +the configured repository path, running checks against a substitute, and +restoring the original path cannot redirect either corpus selection or +validation. + +Each validation tool also starts from an empty process environment. The runner +admits only the ambient executable search path needed to locate the pinned +tools and the `C` locale needed for deterministic diagnostics. Variables such +as `NODE_OPTIONS`, Git repository overrides, runtime preload hooks, and +host-specific configuration cannot enter validation-tool execution. + +Each selected source also retains its device, inode, size, modification time, +and change time. The Rust boundary reopens every path through the retained +repository capability and compares that identity before and after each +external tool. A source replacement, in-place mutation, or +substitute-then-restore sequence refuses the corpus instead of admitting tool +output from ambiguous bytes. The workflow checker disables `actionlint`'s optional `shellcheck` and `pyflakes` integrations. Neither auxiliary executable is admitted or pinned by @@ -73,6 +122,13 @@ The documentation job has read-only repository permissions and disables checkout credential persistence. It writes tools only beneath `RUNNER_TEMP`. A failed or interrupted installation leaves no authoritative state and requires no recovery; a subsequent job starts from a fresh runner. +The Rust workflow contract requires the exact top-level `contents: read` +permission mapping and refuses `write-all`, write authority, additional scopes, +or omitted permissions before admitting any documentation step. +The job invokes `cargo xtask documentation-refusal-check` after installing the +pinned tools. That named Rust command directly executes the broken-fragment and +invalid-workflow scenarios and requires their exact tool refusals; it does not +depend on a libtest substring filter or accept a zero-test run. ## Review triggers diff --git a/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md new file mode 100644 index 0000000..d5286d0 --- /dev/null +++ b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md @@ -0,0 +1,79 @@ +# Dependency Admission: serde 1.0.229 and serde_json 1.0.151 + +- Status: Accepted for repository-task JSON admission only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [serde-rs/json](https://github.com/serde-rs/json) + +## Admitted use + +Keep admits the exactly pinned `serde_json` 1.0.151 and `serde` 1.0.229 +packages only behind the `xtask` crate's `repository-tasks` feature. They parse +the committed Node documentation-tool manifest and lockfile so the Rust +documentation integrity task can validate their structure and reviewed +dependency versions. A Keep-owned recursive visitor rejects duplicate object +members at every depth before returning a JSON value. + +The dependencies are absent from Keep's published library graph, public API, +content identities, durable formats, and production behavior. No +dependency-owned type crosses out of the private repository-task adapter. + +## Why a dependency is needed + +JSON syntax includes escapes, Unicode, numbers, nested collections, and +duplicate representation details that do not belong in a local partial parser. +The documentation gate needs structural lookup, not canonical identity bytes. +Using a maintained parser keeps malformed input fail-closed without creating a +second JSON implementation inside Keep. + +The parsed values are never hashed, persisted, or admitted as Keep domain +types. The repository task reads fixed paths through the bounded, +capability-relative, no-follow file boundary before parsing them. + +## Features and resolved graph + +Both direct dependencies disable default features and enable only `std`. They +are optional and activated solely by `repository-tasks`. + +The active normal dependency graph introduced for this boundary consists of: + +- `itoa` 1.0.18; +- `memchr` 2.8.3; +- `serde` and `serde_core` 1.0.229; and +- `zmij` 1.0.23. + +Cargo's all-target resolution also retains `serde_derive` 1.0.229 and `syn` +3.0.3. Their procedural-macro dependencies were already present in the +workspace lockfile. + +## Safety, licensing, and compatibility + +`serde_json` and `serde` declare the MIT OR Apache-2.0 license expression. +Their manifests declare minimum supported Rust versions below Keep's pinned +toolchain. + +The fuzz-workspace license policy retains Apache-2.0 as its default allowlist +and grants exact-version MIT exceptions to `memchr` 2.8.3 and `zmij` 1.0.23. +Those exceptions admit only the reviewed transitive graph named above; a +resolved version change remains a policy failure until this record and the +exception are reviewed together. + +Keep-owned code invokes only safe APIs. The parser and its transitive +dependencies may contain implementation details outside Keep's `unsafe_code` +lint boundary, so `cargo deny` and RustSec checks remain mandatory +point-in-time evidence. + +## Failure and recovery boundaries + +Malformed JSON, an oversized file, a non-UTF-8 file, a non-regular file, or a +repository-root replacement produces a typed refusal. The task never repairs, +rewrites, or substitutes repository data. Parsing has no durability or +recovery semantics. + +Keep can remove this dependency without changing public or durable behavior by +replacing it with an equally bounded parser that preserves the exact rejection +and structural-validation laws. + +Reopen this admission if the direct version, selected feature, resolved graph, +license, MSRV, repository-task-only boundary, or admitted JSON use changes. diff --git a/docs/dependencies/signal-hook-0.4.4.md b/docs/dependencies/signal-hook-0.4.4.md new file mode 100644 index 0000000..68d1e77 --- /dev/null +++ b/docs/dependencies/signal-hook-0.4.4.md @@ -0,0 +1,81 @@ +# Dependency Admission: signal-hook 0.4.4 + +- Status: Accepted for repository-task terminal-signal handling only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [rust-cli/signal-hook](https://github.com/rust-cli/signal-hook) + +## Admitted use + +Keep admits the exactly pinned `signal-hook` 0.4.4 package only behind the +`xtask` crate's `repository-tasks` feature. The private bounded-process adapter +uses its iterator API to observe `SIGINT`, `SIGTERM`, `SIGHUP`, and `SIGQUIT` +on a dedicated thread while external repository tools are active. + +The adapter records the first signal as a typed interruption. Existing +process-group cleanup then terminates and reaps the child and its descendants. +When no repository tool is active, the adapter invokes the operating system's +default signal behavior. The +[terminal signal process-group decision](../adr/0007-terminal-signal-process-group-guard.md) +records the process-wide concurrency and cleanup policy. + +The package is absent from Keep's published library graph, public API, content +identities, durable formats, and production storage behavior. No +dependency-owned type crosses out of the private repository-task adapter. + +## Why the standard library is insufficient + +The standard library exposes Unix signal constants through platform APIs but +does not provide a safe registration and delivery mechanism for process +signals. Implementing one locally would require unsafe signal-handler code, +async-signal-safety analysis, self-pipe or equivalent wakeup machinery, handler +composition, and platform-specific restoration behavior. + +The admitted package owns that unsafe operating-system boundary. Keep-owned +code consumes signals only through its safe iterator and default-handler APIs. + +## Features and resolved graph + +The direct dependency disables default features and enables only `iterator`. +That feature also enables `channel`. Both are activated solely by +`repository-tasks`. + +The locked graph introduced for this boundary is: + +- `signal-hook` 0.4.4; +- `signal-hook-registry` 1.4.8; +- `errno` 0.3.14; and +- `libc` 0.2.186. + +The latter two packages were already present in the locked workspace graph. + +## Safety, licensing, and compatibility + +`signal-hook` 0.4.4 and `signal-hook-registry` 1.4.8 declare +`MIT OR Apache-2.0`. Their declared Rust-version floors are 1.66 and 1.26, +respectively. Compatibility remains established by Keep's pinned stable, MSRV, +debug, release, Clippy, dependency-policy, and advisory lanes. + +The packages contain unsafe code around operating-system signal registration +and delivery. Keep-owned code invokes only safe APIs, performs no work in a +signal handler, retains no dependency-owned public type, and confines global +registration to the private repository-task adapter. `cargo deny` and RustSec +checks remain mandatory point-in-time evidence. + +## Failure and recovery boundaries + +Registration, iterator creation, or signal-thread creation failure is a typed +refusal before the external command starts. Registry poisoning also refuses +the task. Once a signal is observed, process-group cleanup remains authoritative +and any cleanup failure is preserved alongside the interruption. + +No repository, durable, or authoritative application state is written by this +boundary. Keep can remove the dependency without changing public or durable +behavior by replacing it with an equally portable safe mechanism that +preserves default behavior outside active tasks, typed interruption, bounded +wakeup, concurrent-operation handling, and whole-process-group cleanup. + +Reopen this admission if the direct version, selected features, resolved graph, +license, supported signals, process-global handler policy, or +repository-task-only boundary changes. diff --git a/docs/dependencies/yaml-rust2-0.11.0.md b/docs/dependencies/yaml-rust2-0.11.0.md new file mode 100644 index 0000000..e189066 --- /dev/null +++ b/docs/dependencies/yaml-rust2-0.11.0.md @@ -0,0 +1,68 @@ +# Dependency Admission: yaml-rust2 0.11.0 + +- Status: Accepted for repository-task workflow admission only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [Ethiraric/yaml-rust2](https://github.com/Ethiraric/yaml-rust2) + +## Admitted use + +Keep admits exactly pinned `yaml-rust2` 0.11.0 only behind the `xtask` crate's +`repository-tasks` feature. The documentation-integrity task parses +`.github/workflows/ci.yml`, selects the `documentation` job's actual `run` +fields, and admits only the reviewed command set. Comments, display strings, +unrelated fields, and additional shell commands cannot satisfy that execution +contract. + +The dependency is absent from Keep's published library graph, public API, +content identities, durable formats, and production behavior. Its typed parse +error remains inside the private repository-task adapter. + +## Why a dependency is needed + +YAML includes quoted and block scalars, comments, aliases, nested collections, +and duplicate mapping keys. A substring scan cannot distinguish executable +`run` fields from inert text. A maintained parser keeps the workflow boundary +structural and fail-closed without creating a partial YAML implementation +inside Keep. + +The parsed values are never hashed, persisted, or admitted as Keep domain +types. The task reads the fixed workflow path through the bounded, +capability-relative, no-follow repository-file boundary before parsing it. + +## Features and resolved graph + +The direct dependency disables default features and is optional. It is +activated solely by `repository-tasks`; disabling defaults excludes the +optional non-UTF-8 input support. + +The introduced normal dependency graph is: + +- `arraydeque` 0.5.1; +- `foldhash` 0.2.0; +- `hashbrown` 0.16.1; and +- `hashlink` 0.11.1. + +## Safety, licensing, and compatibility + +`yaml-rust2` declares the MIT OR Apache-2.0 license expression and a minimum +supported Rust version of 1.65, below Keep's pinned toolchain. Its 0.11.0 Rust +source contains no `unsafe` block. Keep-owned code invokes only safe APIs. + +`cargo deny check licenses bans sources` and `cargo audit` pass with the +resolved graph. These checks remain mandatory point-in-time evidence. + +## Failure and recovery boundaries + +Malformed YAML, duplicate mapping keys, an absent documentation job, an +unreviewed command, an oversized workflow, a non-UTF-8 workflow, or a replaced +repository root produces a typed refusal. The task never repairs, rewrites, or +substitutes workflow data. Parsing has no durability or recovery semantics. + +Keep can remove this dependency without changing public or durable behavior by +replacing it with an equally bounded parser that preserves structural `run` +selection, duplicate-key refusal, and the reviewed-command laws. + +Reopen this admission if the direct version, selected features, resolved graph, +license, MSRV, repository-task-only boundary, or admitted YAML use changes. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 0b03751..7d8e423 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -84,6 +84,12 @@ dependencies = [ "r-efi", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "jobserver" version = "0.1.35" @@ -126,18 +132,111 @@ dependencies = [ "cc", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + [[package]] name = "xtask" version = "0.0.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b708353..bb29289 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,7 +12,7 @@ cargo-fuzz = true [dependencies] keep = { path = "..", version = "=0.0.0" } libfuzzer-sys = { version = "=0.4.13", default-features = false, features = ["link_libfuzzer"] } -xtask = { path = "../xtask", version = "=0.0.0", default-features = false, features = ["golden-protocol-fuzz"] } +xtask = { path = "../xtask", version = "=0.0.0", default-features = false, features = ["golden-protocol-fuzz", "repository-json-fuzz"] } [lints.rust] warnings = "deny" @@ -110,6 +110,13 @@ test = false doc = false bench = false +[[bin]] +name = "repository_json" +path = "fuzz_targets/repository_json.rs" +test = false +doc = false +bench = false + [[bin]] name = "segment_format" path = "fuzz_targets/segment_format.rs" diff --git a/fuzz/README.md b/fuzz/README.md index e900d6c..4cb71eb 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -12,6 +12,13 @@ versions and resource limits. `cargo xtask fuzz` parses that file without shell substitution and refuses missing, duplicate, unknown, malformed, or out-of-bound values. +The external `cargo fuzz build` process uses the reviewed build timeout. Each +`cargo fuzz run` process uses its profile's exploration budget plus the +reviewed process-grace interval; the addition is checked before execution. +Expiry terminates and reaps the complete child process group. The grace +interval covers Cargo startup and libFuzzer shutdown without enlarging +libFuzzer's exploration budget. + The Rust task compares `cargo fuzz list` with the checked-in target files, sorts the exact target names, bounds child-process output, and exercises every target even if an earlier target fails. Run: @@ -42,6 +49,12 @@ schemas plus canonical case, canonical decimal, invalid-identity classification, and mutation parsing under the campaign's one-mebibyte input bound. +The `repository_json` target feeds arbitrary bytes through the same +duplicate-refusing parser used for the documentation-tool manifest and lock +graph. Its facade refuses inputs over one mebibyte before parsing; integration +laws cover valid nested JSON, malformed bytes, excessive nesting, and duplicate +members below an enclosing object and array. + The `layout_record` seeds are the four exact canonical binary records derived from the reviewed hexadecimal fixtures. The target feeds arbitrary bytes through the bounded decoder and, for every admitted record, requires exact diff --git a/fuzz/campaign.env b/fuzz/campaign.env index b5025cd..326dbe8 100644 --- a/fuzz/campaign.env +++ b/fuzz/campaign.env @@ -8,9 +8,11 @@ FUZZ_MAX_INPUT_BYTES=1048576 FUZZ_RSS_LIMIT_MB=1024 # Campaign-specific exploration budgets. +FUZZ_BUILD_TIMEOUT_SECONDS=600 FUZZ_SMOKE_SECONDS_PER_TARGET=15 FUZZ_SCHEDULED_SECONDS_PER_TARGET=600 FUZZ_CMIN_SECONDS_PER_TARGET=120 +FUZZ_PROCESS_GRACE_SECONDS=60 # Retained derived-corpus bounds. FUZZ_CORPUS_MAX_FILES=20000 diff --git a/fuzz/deny.toml b/fuzz/deny.toml index 9e12454..5eae9d4 100644 --- a/fuzz/deny.toml +++ b/fuzz/deny.toml @@ -16,6 +16,10 @@ exceptions = [ { allow = ["BSD-2-Clause"], crate = "arrayref@0.3.9" }, # `libfuzzer-sys` is confined to this non-published fuzz workspace. { allow = ["MIT", "NCSA"], crate = "libfuzzer-sys@0.4.13" }, + # `memchr` is a reviewed transitive dependency of locked serde_json 1.0.151. + { allow = ["MIT"], crate = "memchr@2.8.3" }, + # `zmij` is a reviewed transitive dependency of locked serde_json 1.0.151. + { allow = ["MIT"], crate = "zmij@1.0.23" }, ] [sources] diff --git a/fuzz/fuzz_targets/repository_json.rs b/fuzz/fuzz_targets/repository_json.rs new file mode 100644 index 0000000..ed85dfe --- /dev/null +++ b/fuzz/fuzz_targets/repository_json.rs @@ -0,0 +1,10 @@ +#![no_main] + +//! This target owns bounded duplicate-refusing repository JSON fuzzing. + +use libfuzzer_sys::fuzz_target; +use xtask::admit_repository_json; + +fuzz_target!(|bytes: &[u8]| { + let _ = admit_repository_json(bytes); +}); diff --git a/repository-process-spawn/Cargo.toml b/repository-process-spawn/Cargo.toml new file mode 100644 index 0000000..1e6a5e4 --- /dev/null +++ b/repository-process-spawn/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "repository-process-spawn" +version = "0.0.0" +edition = "2024" +rust-version = "1.96" +license = "Apache-2.0" +publish = false + +[dependencies] +rustix = { version = "=1.1.4", default-features = false, features = ["process", "std"] } + +[lints] +workspace = true diff --git a/repository-process-spawn/src/lib.rs b/repository-process-spawn/src/lib.rs new file mode 100644 index 0000000..978492d --- /dev/null +++ b/repository-process-spawn/src/lib.rs @@ -0,0 +1,34 @@ +//! Isolated child-process setup for descriptor-bound working directories. +//! +//! This crate contains Keep's only admitted unsafe boundary. It exists because +//! [`std::process::Command`] exposes child setup through an unsafe hook, while +//! an opened directory descriptor is required to prevent pathname-replacement +//! races. No filesystem, storage, identity, or durable-format code belongs +//! here. + +#![allow( + unsafe_code, + reason = "this dedicated crate owns the reviewed pre-exec fchdir boundary" +)] + +use std::os::fd::OwnedFd; +use std::os::unix::process::CommandExt; +use std::process::Command; + +/// Configures `command` to enter `directory` after fork and before exec. +/// +/// The descriptor remains owned by the command hook. Any `fchdir` failure is +/// returned by the later spawn operation. The hook performs no allocation, +/// locking, I/O buffering, or user callback. +pub fn set_working_directory(command: &mut Command, directory: OwnedFd) { + // SAFETY: POSIX specifies fchdir as async-signal-safe. The hook captures an + // owned descriptor and performs exactly that operation between fork and + // exec. It does not allocate, lock, inspect ambient paths, or call user + // code. Rustix supplies the checked safe wrapper around the system call. + unsafe { + command.pre_exec(move || { + rustix::process::fchdir(&directory)?; + Ok(()) + }); + } +} diff --git a/repository-process-spawn/tests/working_directory.rs b/repository-process-spawn/tests/working_directory.rs new file mode 100644 index 0000000..da471c7 --- /dev/null +++ b/repository-process-spawn/tests/working_directory.rs @@ -0,0 +1,76 @@ +//! Integration laws for descriptor-bound child working directories. + +use std::env; +use std::fs::{self, File}; +use std::io; +use std::os::fd::OwnedFd; +use std::path::PathBuf; +use std::process::{self, Command}; + +use repository_process_spawn::set_working_directory; + +#[test] +fn child_uses_the_opened_directory_without_mutating_parent_state() +-> Result<(), Box> { + let parent = env::current_dir()?; + let world = test_world("replacement"); + let root = world.join("repository"); + let retained = world.join("retained"); + fs::create_dir_all(&root)?; + fs::write(root.join("marker"), b"original\n")?; + let directory: OwnedFd = File::open(&root)?.into(); + + fs::rename(&root, &retained)?; + fs::create_dir(&root)?; + fs::write(root.join("marker"), b"substitute\n")?; + + let mut command = Command::new("cat"); + command.arg("marker"); + set_working_directory(&mut command, directory); + let output = command.output()?; + + fs::remove_dir_all(&root)?; + fs::remove_dir_all(&retained)?; + fs::remove_dir(&world)?; + + assert!(output.status.success()); + assert_eq!(output.stdout, b"original\n"); + assert_eq!(env::current_dir()?, parent); + Ok(()) +} + +#[test] +fn non_directory_descriptor_refuses_before_exec() -> Result<(), Box> { + let world = test_world("non-directory"); + fs::create_dir(&world)?; + let file_path = world.join("file"); + fs::write(&file_path, b"not a directory\n")?; + let descriptor: OwnedFd = File::open(&file_path)?.into(); + let mut command = Command::new("true"); + set_working_directory(&mut command, descriptor); + + let result = command.spawn(); + fs::remove_file(&file_path)?; + fs::remove_dir(&world)?; + let error = match result { + Ok(mut child) => { + let termination = child.kill(); + let reap = child.wait(); + return Err(format!( + "non-directory descriptor unexpectedly reached exec; termination: {termination:?}; reap: {reap:?}" + ) + .into()); + } + Err(error) => error, + }; + + assert_eq!(error.kind(), io::ErrorKind::NotADirectory); + Ok(()) +} + +fn test_world(case: &str) -> PathBuf { + env::temp_dir().join(format!( + "keep-repository-process-spawn-{}-{case}", + process::id() + )) +} diff --git a/scripts/check_markdown.py b/scripts/check_markdown.py deleted file mode 100644 index f6cfe9d..0000000 --- a/scripts/check_markdown.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Check the Git-admitted Markdown corpus and its internal links.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -import sys - -EXPECTED_LINTER_VERSION = "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)" -EXPECTED_LINK_CHECKER_VERSION = "lychee 0.21.0" - - -def find_linter() -> str: - """Return the Markdown linter path or report a precise setup failure.""" - executable = shutil.which("markdownlint-cli2") - if executable is None: - raise RuntimeError( - "markdownlint-cli2 is unavailable; install version 0.23.2" - ) - return executable - - -def find_link_checker() -> str: - """Return the link checker path or report a precise setup failure.""" - executable = shutil.which("lychee") - if executable is None: - raise RuntimeError("lychee is unavailable; install version 0.21.0") - return executable - - -def verify_linter_version(executable: str) -> None: - """Refuse a Markdown linter version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "--no-globs", "--version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_LINTER_VERSION: - raise RuntimeError( - f"markdownlint-cli2 version mismatch: " - f"expected {EXPECTED_LINTER_VERSION!r}, observed {observed!r}" - ) - - -def verify_link_checker_version(executable: str) -> None: - """Refuse a link checker version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "--version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_LINK_CHECKER_VERSION: - raise RuntimeError( - f"lychee version mismatch: " - f"expected {EXPECTED_LINK_CHECKER_VERSION!r}, " - f"observed {observed!r}" - ) - - -def admit_source_path(path: str) -> bool: - """Admit one existing regular file without following links.""" - try: - mode = os.lstat(path).st_mode - except FileNotFoundError: - return False - except OSError as error: - raise RuntimeError( - f"cannot inspect Markdown source {path!r}: {error}" - ) from error - if not stat.S_ISREG(mode): - raise RuntimeError(f"Markdown source is not a regular file: {path!r}") - return True - - -def source_markdown() -> list[str]: - """Return tracked and nonignored new Markdown in deterministic order.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - "*.md", - ], - check=False, - stdout=subprocess.PIPE, - ) - if completed.returncode != 0: - raise RuntimeError("git ls-files failed while selecting Markdown") - paths = [] - for raw_path in completed.stdout.split(b"\0"): - if raw_path: - path = os.fsdecode(raw_path) - if admit_source_path(path): - paths.append(path) - paths.sort() - if not paths: - raise RuntimeError("the source Markdown corpus is empty") - return paths - - -def main() -> int: - """Run deterministic Markdown and internal-link checks.""" - try: - linter = find_linter() - link_checker = find_link_checker() - verify_linter_version(linter) - verify_link_checker_version(link_checker) - paths = source_markdown() - except RuntimeError as error: - print(f"Markdown check refused: {error}", file=sys.stderr) - return 1 - - lint_result = subprocess.run( - [linter, "--no-globs", "--", *paths], - check=False, - ) - link_result = subprocess.run( - [ - link_checker, - "--offline", - "--include-fragments", - "--no-progress", - "--format", - "detailed", - "--", - *paths, - ], - check=False, - ) - return lint_result.returncode or link_result.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py deleted file mode 100644 index abb556b..0000000 --- a/scripts/check_workflows.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -"""Check every GitHub Actions workflow with the pinned actionlint tool.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -import sys - -EXPECTED_VERSION = "1.7.12" - - -def find_actionlint() -> str: - """Return the actionlint path or report a precise setup failure.""" - executable = shutil.which("actionlint") - if executable is None: - raise RuntimeError("actionlint is unavailable; install version 1.7.12") - return executable - - -def verify_version(executable: str) -> None: - """Refuse an actionlint version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "-version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_VERSION: - raise RuntimeError( - f"actionlint version mismatch: " - f"expected {EXPECTED_VERSION!r}, observed {observed!r}" - ) - - -def admit_workflow_path(path: str) -> bool: - """Admit one existing regular workflow without following links.""" - try: - mode = os.lstat(path).st_mode - except FileNotFoundError: - return False - except OSError as error: - raise RuntimeError( - f"cannot inspect workflow source {path!r}: {error}" - ) from error - if not stat.S_ISREG(mode): - raise RuntimeError(f"workflow source is not a regular file: {path!r}") - return True - - -def workflow_paths() -> list[str]: - """Return Git-admitted workflow paths in deterministic order.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - ".github/workflows/*.yml", - ".github/workflows/*.yaml", - ], - check=False, - stdout=subprocess.PIPE, - ) - if completed.returncode != 0: - raise RuntimeError("git ls-files failed while selecting workflows") - paths = [] - for raw_path in completed.stdout.split(b"\0"): - if raw_path: - path = os.fsdecode(raw_path) - if admit_workflow_path(path): - paths.append(path) - paths.sort() - if not paths: - raise RuntimeError("the GitHub Actions workflow corpus is empty") - return paths - - -def main() -> int: - """Run actionlint against the deterministic workflow input boundary.""" - try: - executable = find_actionlint() - verify_version(executable) - paths = workflow_paths() - except RuntimeError as error: - print(f"Workflow check refused: {error}", file=sys.stderr) - return 1 - - completed = subprocess.run( - [executable, "-shellcheck=", "-pyflakes=", *paths], - check=False, - ) - return completed.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/test_check_markdown.py b/scripts/test_check_markdown.py deleted file mode 100644 index c5a4755..0000000 --- a/scripts/test_check_markdown.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Laws for the repository-owned Markdown source boundary.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from check_markdown import source_markdown - -REPOSITORY_ROOT = Path(__file__).parent.parent - - -class MarkdownCorpusLaws(unittest.TestCase): - """The selected Markdown corpus depends only on admitted source files.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - self.run_git("init", "--quiet") - - def run_git(self, *arguments: str) -> None: - """Run one Git setup command inside the isolated repository.""" - subprocess.run( - ["git", *arguments], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - def test_repository_ignored_markdown_cannot_enter_the_corpus(self) -> None: - self.write(".gitignore", "/target/\n") - self.write("tracked.md", "# Tracked\n") - self.write("new.md", "# New\n") - self.write("target/generated.md", "# Generated\n") - self.run_git("add", ".gitignore", "tracked.md") - - self.assertEqual(source_markdown(), ["new.md", "tracked.md"]) - - def test_source_paths_have_canonical_lexical_order(self) -> None: - self.write("zulu.md", "# Zulu\n") - self.write("alpha.md", "# Alpha\n") - self.run_git("add", "zulu.md", "alpha.md") - - self.assertEqual(source_markdown(), ["alpha.md", "zulu.md"]) - - def test_deleted_tracked_markdown_is_not_forwarded(self) -> None: - self.write("deleted.md", "# Deleted\n") - self.write("remaining.md", "# Remaining\n") - self.run_git("add", "deleted.md", "remaining.md") - (self.root / "deleted.md").unlink() - - self.assertEqual(source_markdown(), ["remaining.md"]) - - def test_user_global_ignores_cannot_change_the_corpus(self) -> None: - self.write("tracked.md", "# Tracked\n") - self.write("new.md", "# New\n") - self.write("global-ignore", "*.md\n") - self.run_git("add", "tracked.md") - global_config = self.root / "global.gitconfig" - subprocess.run( - [ - "git", - "config", - "--file", - str(global_config), - "core.excludesFile", - str(self.root / "global-ignore"), - ], - check=True, - capture_output=True, - ) - - with patch.dict( - os.environ, - { - "GIT_CONFIG_GLOBAL": str(global_config), - "GIT_CONFIG_NOSYSTEM": "1", - }, - ): - self.assertEqual(source_markdown(), ["new.md", "tracked.md"]) - - def test_symlinked_markdown_is_refused(self) -> None: - self.write("target/generated.md", "# Generated\n") - (self.root / "linked.md").symlink_to("target/generated.md") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - source_markdown() - - def test_fifo_markdown_is_refused(self) -> None: - self.write("blocking.md", "# Initially regular\n") - self.run_git("add", "blocking.md") - (self.root / "blocking.md").unlink() - os.mkfifo(self.root / "blocking.md") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - source_markdown() - - -class DocumentationCommandLaws(unittest.TestCase): - """Contributor commands inspect changes that have not been committed.""" - - def test_whitespace_checks_cover_the_index_and_working_tree(self) -> None: - for relative_path in ( - "CONTRIBUTING.md", - "docs/Documentation Standards.md", - ): - source = (REPOSITORY_ROOT / relative_path).read_text( - encoding="utf-8" - ) - self.assertNotIn( - 'git diff --check "$(git hash-object -t tree /dev/null)" HEAD', - source, - ) - self.assertIn("git diff --check\n", source) - self.assertIn("git diff --cached --check\n", source) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_dependabot_coverage.py b/scripts/test_dependabot_coverage.py deleted file mode 100644 index 0353453..0000000 --- a/scripts/test_dependabot_coverage.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Laws for complete, uniform Dependabot manifest coverage.""" - -from __future__ import annotations - -import os -import subprocess -import unittest -from pathlib import Path - -REPOSITORY_ROOT = Path(__file__).parent.parent -CONFIG_PATH = REPOSITORY_ROOT / ".github" / "dependabot.yml" -UPDATE_MARKER = " - package-ecosystem: " -MANIFEST_ECOSYSTEMS = { - "Cargo.toml": "cargo", - "package.json": "npm", -} -STATIC_SCOPES = {("github-actions", "/")} - - -def update_blocks(raw: str) -> list[str]: - """Return each Dependabot update block in source order.""" - lines = raw.splitlines() - starts = [ - index - for index, line in enumerate(lines) - if line.startswith(UPDATE_MARKER) - ] - return [ - "\n".join(lines[start:end]) - for start, end in zip(starts, [*starts[1:], len(lines)], strict=True) - ] - - -def unquote(raw: str) -> str: - """Remove one matching YAML scalar quote pair.""" - if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: - return raw[1:-1] - return raw - - -def configured_scopes(blocks: list[str]) -> list[tuple[str, str]]: - """Return every configured ecosystem and manifest directory pair.""" - scopes: list[tuple[str, str]] = [] - for block in blocks: - lines = block.splitlines() - ecosystem = unquote(lines[0].removeprefix(UPDATE_MARKER)) - for index, line in enumerate(lines): - if line.startswith(" directory: "): - scopes.append( - (ecosystem, unquote(line.removeprefix(" directory: "))) - ) - if line == " directories:": - for directory_line in lines[index + 1 :]: - if not directory_line.startswith(" - "): - break - scopes.append( - ( - ecosystem, - unquote(directory_line.removeprefix(" - ")), - ) - ) - return scopes - - -def tracked_manifest_scopes() -> set[tuple[str, str]]: - """Return dependency scopes implied by tracked first-party manifests.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--", - ":(glob)**/Cargo.toml", - ":(glob)**/package.json", - ], - cwd=REPOSITORY_ROOT, - check=True, - stdout=subprocess.PIPE, - ) - scopes = set() - for raw_path in completed.stdout.split(b"\0"): - if not raw_path: - continue - path = Path(os.fsdecode(raw_path)) - ecosystem = MANIFEST_ECOSYSTEMS[path.name] - directory = ( - "/" - if path.parent == Path(".") - else f"/{path.parent.as_posix()}" - ) - scopes.add((ecosystem, directory)) - return scopes.union(STATIC_SCOPES) - - -class DependabotCoverageLaws(unittest.TestCase): - """Every first-party manifest receives one uniform update policy.""" - - def setUp(self) -> None: - self.raw = CONFIG_PATH.read_text(encoding="utf-8") - self.blocks = update_blocks(self.raw) - - def test_every_dependency_manifest_has_an_update_scope(self) -> None: - observed = set(configured_scopes(self.blocks)) - self.assertEqual(tracked_manifest_scopes().difference(observed), set()) - - def test_update_scopes_are_unique(self) -> None: - scopes = configured_scopes(self.blocks) - self.assertEqual(len(scopes), len(set(scopes))) - - def test_every_update_uses_the_maintenance_policy(self) -> None: - self.assertTrue(self.raw.startswith("version: 2\nupdates:\n")) - self.assertTrue(self.blocks) - for block in self.blocks: - self.assertIn(" schedule:\n interval: weekly", block) - self.assertIn(" open-pull-requests-limit: 5", block) - self.assertIn(" labels:\n - dependencies", block) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_documentation_integrity.py b/scripts/test_documentation_integrity.py deleted file mode 100644 index 200290a..0000000 --- a/scripts/test_documentation_integrity.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Negative laws for the required documentation integrity gates.""" - -from __future__ import annotations - -import json -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -import check_markdown -import check_workflows - -REPOSITORY_ROOT = Path(__file__).parent.parent -LAW_COMMAND = "python3 -m unittest discover -s scripts -p 'test_*.py' -v" - - -class ToolVersionLaws(unittest.TestCase): - """Every required checker refuses an unexpected executable version.""" - - def test_wrong_markdownlint_version_is_refused(self) -> None: - with patch.object( - check_markdown.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="markdownlint-cli2 v999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_markdown.verify_linter_version("markdownlint-cli2") - - def test_wrong_lychee_version_is_refused(self) -> None: - with patch.object( - check_markdown.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="lychee 999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_markdown.verify_link_checker_version("lychee") - - def test_wrong_actionlint_version_is_refused(self) -> None: - with patch.object( - check_workflows.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_workflows.verify_version("actionlint") - - -class WorkflowContractLaws(unittest.TestCase): - """Required CI executes the repository's negative integrity laws.""" - - def test_documentation_job_runs_negative_integrity_laws(self) -> None: - workflow = ( - REPOSITORY_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - self.assertIn(LAW_COMMAND, workflow) - - def test_actionlint_disables_unadmitted_auxiliary_linters(self) -> None: - completed = SimpleNamespace(returncode=0) - with ( - patch.object( - check_workflows, - "find_actionlint", - return_value="actionlint", - ), - patch.object(check_workflows, "verify_version"), - patch.object( - check_workflows, - "workflow_paths", - return_value=[".github/workflows/ci.yml"], - ), - patch.object( - check_workflows.subprocess, - "run", - return_value=completed, - ) as run, - ): - self.assertEqual(check_workflows.main(), 0) - - run.assert_called_once_with( - [ - "actionlint", - "-shellcheck=", - "-pyflakes=", - ".github/workflows/ci.yml", - ], - check=False, - ) - - -class ToolInstallerLaws(unittest.TestCase): - """The Markdown tool graph is fully locked before network installation.""" - - def test_patched_parser_is_directly_admitted_without_override(self) -> None: - tool_directory = ( - REPOSITORY_ROOT / "scripts" / "documentation-tools" - ) - manifest = json.loads( - (tool_directory / "package.json").read_text(encoding="utf-8") - ) - lock = json.loads( - (tool_directory / "package-lock.json").read_text(encoding="utf-8") - ) - - self.assertNotIn("overrides", manifest) - self.assertEqual( - lock["packages"]["node_modules/markdownlint-cli2"][ - "dependencies" - ]["js-yaml"], - "5.2.2", - ) - - def test_known_parser_denial_of_service_versions_are_refused(self) -> None: - lock_path = ( - REPOSITORY_ROOT - / "scripts" - / "documentation-tools" - / "package-lock.json" - ) - packages = json.loads(lock_path.read_text(encoding="utf-8"))[ - "packages" - ] - self.assertEqual( - packages["node_modules/js-yaml"]["version"], - "5.2.2", - ) - self.assertEqual( - packages["node_modules/markdown-it"]["version"], - "14.3.0", - ) - - def test_markdown_dependency_graph_is_lockfile_admitted(self) -> None: - tool_directory = ( - REPOSITORY_ROOT / "scripts" / "documentation-tools" - ) - lock_path = tool_directory / "package-lock.json" - self.assertTrue(lock_path.is_file()) - lock = json.loads(lock_path.read_text(encoding="utf-8")) - self.assertEqual(lock["lockfileVersion"], 3) - self.assertEqual( - lock["packages"][""]["dependencies"]["markdownlint-cli2"], - "0.23.2", - ) - for path, package in lock["packages"].items(): - if path: - self.assertIn("resolved", package, path) - self.assertIn("integrity", package, path) - - installer = ( - REPOSITORY_ROOT / "scripts" / "install_documentation_tools.sh" - ).read_text(encoding="utf-8") - self.assertIn("npm ci", installer) - self.assertIn("package-lock.json", installer) - self.assertNotIn("npm install \\", installer) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_documentation_refusals.py b/scripts/test_documentation_refusals.py deleted file mode 100644 index bc357cf..0000000 --- a/scripts/test_documentation_refusals.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Precise refusal laws for malformed documentation and workflows.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path - -import check_markdown -import check_workflows - - -class IsolatedRepositoryTestCase(unittest.TestCase): - """Own one temporary Git repository for a malformed source fixture.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - subprocess.run( - ["git", "init", "--quiet"], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - -class IntegrityRefusalLaws(IsolatedRepositoryTestCase): - """Malformed inputs produce the expected checker failure class.""" - - def test_broken_internal_fragment_is_refused(self) -> None: - self.write( - "source.md", - "# Source\n\n[Missing](target.md#missing-heading)\n", - ) - self.write("target.md", "# Present heading\n") - subprocess.run( - ["git", "add", "source.md", "target.md"], - check=True, - capture_output=True, - ) - linter = check_markdown.find_linter() - link_checker = check_markdown.find_link_checker() - check_markdown.verify_linter_version(linter) - check_markdown.verify_link_checker_version(link_checker) - paths = check_markdown.source_markdown() - - lint = subprocess.run( - [linter, "--no-globs", "--", *paths], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(lint.returncode, 0, lint.stderr) - links = subprocess.run( - [ - link_checker, - "--offline", - "--include-fragments", - "--no-progress", - "--format", - "detailed", - "--", - *paths, - ], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(links.returncode, 2) - self.assertIn( - "Cannot find fragment", - f"{links.stdout}\n{links.stderr}", - ) - - def test_invalid_workflow_is_refused(self) -> None: - self.write( - ".github/workflows/invalid.yml", - "name: Invalid\non: [push\n", - ) - actionlint = check_workflows.find_actionlint() - check_workflows.verify_version(actionlint) - paths = check_workflows.workflow_paths() - - completed = subprocess.run( - [actionlint, "-shellcheck=", "-pyflakes=", *paths], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(completed.returncode, 1) - self.assertIn( - "could not parse as YAML", - f"{completed.stdout}\n{completed.stderr}", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_workflow_corpus.py b/scripts/test_workflow_corpus.py deleted file mode 100644 index 8531131..0000000 --- a/scripts/test_workflow_corpus.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Laws for the repository-owned GitHub Actions source boundary.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from check_workflows import workflow_paths - - -class WorkflowCorpusLaws(unittest.TestCase): - """The selected workflow corpus contains only admitted source files.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - self.run_git("init", "--quiet") - - def run_git(self, *arguments: str) -> None: - """Run one Git setup command inside the isolated repository.""" - subprocess.run( - ["git", *arguments], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - def test_repository_ignored_workflow_cannot_enter_the_corpus(self) -> None: - self.write(".gitignore", "/.github/workflows/generated.yml\n") - self.write(".github/workflows/tracked.yml", "name: Tracked\n") - self.write(".github/workflows/new.yml", "name: New\n") - self.write(".github/workflows/generated.yml", "name: Generated\n") - self.run_git( - "add", - ".gitignore", - ".github/workflows/tracked.yml", - ) - - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/new.yml", - ".github/workflows/tracked.yml", - ], - ) - - def test_source_paths_have_one_canonical_lexical_order(self) -> None: - self.write(".github/workflows/zulu.yml", "name: Zulu\n") - self.write(".github/workflows/alpha.yaml", "name: Alpha\n") - self.run_git( - "add", - ".github/workflows/zulu.yml", - ".github/workflows/alpha.yaml", - ) - - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/alpha.yaml", - ".github/workflows/zulu.yml", - ], - ) - - def test_deleted_tracked_workflow_is_not_forwarded(self) -> None: - self.write(".github/workflows/deleted.yml", "name: Deleted\n") - self.write(".github/workflows/remaining.yml", "name: Remaining\n") - self.run_git( - "add", - ".github/workflows/deleted.yml", - ".github/workflows/remaining.yml", - ) - (self.root / ".github/workflows/deleted.yml").unlink() - - self.assertEqual( - workflow_paths(), - [".github/workflows/remaining.yml"], - ) - - def test_user_global_ignores_cannot_change_the_corpus(self) -> None: - self.write(".github/workflows/tracked.yml", "name: Tracked\n") - self.write(".github/workflows/new.yml", "name: New\n") - self.write("global-ignore", "*.yml\n") - self.run_git("add", ".github/workflows/tracked.yml") - global_config = self.root / "global.gitconfig" - subprocess.run( - [ - "git", - "config", - "--file", - str(global_config), - "core.excludesFile", - str(self.root / "global-ignore"), - ], - check=True, - capture_output=True, - ) - - with patch.dict( - os.environ, - { - "GIT_CONFIG_GLOBAL": str(global_config), - "GIT_CONFIG_NOSYSTEM": "1", - }, - ): - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/new.yml", - ".github/workflows/tracked.yml", - ], - ) - - def test_symlinked_workflow_is_refused(self) -> None: - self.write("generated.yml", "name: Generated\n") - workflow_dir = self.root / ".github/workflows" - workflow_dir.mkdir(parents=True) - (workflow_dir / "linked.yml").symlink_to("../../generated.yml") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - workflow_paths() - - def test_fifo_workflow_cannot_enter_the_corpus(self) -> None: - self.write(".github/workflows/regular.yml", "name: Regular\n") - workflow_dir = self.root / ".github/workflows" - os.mkfifo(workflow_dir / "blocking.yml") - - self.assertEqual( - workflow_paths(), - [".github/workflows/regular.yml"], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index afb0695..fee2de4 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,7 +9,18 @@ publish = false [features] default = ["repository-tasks"] golden-protocol-fuzz = [] -repository-tasks = ["dep:blake3", "dep:cap-fs-ext", "dep:cap-std", "dep:md-5"] +repository-json-fuzz = ["dep:serde", "dep:serde_json"] +repository-tasks = [ + "dep:blake3", + "dep:cap-fs-ext", + "dep:cap-std", + "dep:md-5", + "dep:repository-process-spawn", + "dep:rustix", + "repository-json-fuzz", + "dep:signal-hook", + "dep:yaml-rust2", +] [dependencies] # Independent Golden File Worldline oracle recomputes BLAKE3 corpus witnesses. @@ -19,6 +30,18 @@ cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], cap-std = { version = "=4.0.2", default-features = false, optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } +# Dedicated unsafe boundary sets exact child working directories by descriptor. +repository-process-spawn = { path = "../repository-process-spawn", optional = true } +# Safe POSIX process-group signaling bounds failed repository-tool subprocesses. +rustix = { version = "=1.1.4", default-features = false, features = ["process", "std"], optional = true } +# Serde drives duplicate-refusing repository JSON admission; no types escape xtask. +serde = { version = "=1.0.229", default-features = false, features = ["std"], optional = true } +# Typed JSON admission checks the committed documentation-tool lock graph. +serde_json = { version = "=1.0.151", default-features = false, features = ["std"], optional = true } +# Safe self-pipe delivery forwards terminal signals to active child groups. +signal-hook = { version = "=0.4.4", default-features = false, features = ["iterator"], optional = true } +# Pure Rust YAML admission identifies executable GitHub Actions steps. +yaml-rust2 = { version = "=0.11.0", default-features = false, optional = true } [[bin]] name = "xtask" diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs new file mode 100644 index 0000000..25e8ea7 --- /dev/null +++ b/xtask/src/bounded_process.rs @@ -0,0 +1,70 @@ +//! This module owns bounded external child-process collection. + +mod capture; +mod capture_limit; +mod cleanup; +mod deadline; +mod error; +mod interrupt; +mod process_group; +mod reader; + +use std::os::unix::process::CommandExt; +use std::process::Command; +use std::time::Duration; + +use capture::wait_for_child; +pub(crate) use capture::{capture, capture_with, capture_with_limits}; +pub(crate) use capture_limit::CaptureLimits; +use deadline::ProcessDeadline; +pub(crate) use error::ProcessError; +use interrupt::InterruptGuard; +use reader::ReaderWorker; + +/// The completed child status and any output retained by the selected mode. +/// +/// Captured execution retains at most the selected limit for each output +/// stream. Inherited execution leaves both byte vectors empty because the child +/// writes directly to the parent's configured streams. +pub(crate) struct ProcessOutput { + /// The platform exit code, or `None` when the process ended by signal. + pub(crate) code: Option, + /// Whether the platform status represents successful termination. + pub(crate) succeeded: bool, + /// Captured standard output, or an empty vector in inherited mode. + pub(crate) stdout: Vec, + /// Captured standard error, or an empty vector in inherited mode. + pub(crate) stderr: Vec, +} + +/// Runs a child synchronously while inheriting its configured output streams. +/// +/// The optional deadline bounds the complete wait. The child starts in a +/// dedicated process group so timeout or polling failure terminates descendants +/// before this function returns. Spawn, poll, wait, timeout, and cleanup +/// failures retain their typed [`ProcessError`] boundary. +pub(crate) fn status( + program: &'static str, + command: &mut Command, + deadline: Option, +) -> Result { + let deadline = ProcessDeadline::new(program, deadline)?; + let interrupts = InterruptGuard::begin(program)?; + command.process_group(0); + let mut child = command.spawn().map_err(|source| ProcessError::Io { + program, + action: "spawn", + source, + })?; + let status = wait_for_child(program, &mut child, &deadline, &interrupts)?; + Ok(ProcessOutput { + code: status.code(), + succeeded: status.success(), + stdout: Vec::new(), + stderr: Vec::new(), + }) +} + +#[cfg(test)] +#[path = "bounded_process/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs new file mode 100644 index 0000000..a3f7ff5 --- /dev/null +++ b/xtask/src/bounded_process/capture.rs @@ -0,0 +1,210 @@ +//! This module owns bounded child-process capture and collection. + +use std::os::unix::process::CommandExt; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::thread; +use std::time::Duration; + +use super::cleanup::{cleanup_process, join_after_cleanup, join_readers}; +use super::{ + CaptureLimits, InterruptGuard, ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker, +}; +use crate::process_output::BoundedBytes; + +const DEFAULT_CAPTURE_LIMITS: CaptureLimits = CaptureLimits::new(1_048_576, 1_048_576); +const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(10); + +/// Runs a child synchronously and captures bounded standard output and error. +/// +/// Each stream is drained concurrently and retains at most one mebibyte. The +/// optional deadline covers child execution and reader collection. Failures +/// terminate the child's dedicated process group, join both readers, and retain +/// the primary and cleanup errors in [`ProcessError`]. +pub(crate) fn capture( + program: &'static str, + command: &mut Command, + deadline: Option, +) -> Result { + capture_with(program, command, deadline, Command::spawn) +} + +/// Runs one captured child through an injected spawn boundary. +/// +/// This variant retains the default one-mebibyte limit for each stream while +/// allowing a capability-bound caller to own the actual spawn operation. +pub(crate) fn capture_with( + program: &'static str, + command: &mut Command, + deadline: Option, + spawn: impl FnOnce(&mut Command) -> Result, +) -> Result { + capture_with_limits(program, command, deadline, DEFAULT_CAPTURE_LIMITS, spawn) +} + +/// Runs one captured child with exact independent stream limits. +/// +/// The deadline covers child execution and both reader workers. Every failure +/// terminates the dedicated process group and joins the workers before return. +pub(crate) fn capture_with_limits( + program: &'static str, + command: &mut Command, + deadline: Option, + limits: CaptureLimits, + spawn: impl FnOnce(&mut Command) -> Result, +) -> Result { + let deadline = ProcessDeadline::new(program, deadline)?; + let interrupts = InterruptGuard::begin(program)?; + CapturedProcess::start(program, command, spawn, interrupts, limits)?.finish(program, &deadline) +} + +struct CapturedProcess { + child: Child, + interrupts: InterruptGuard, + limits: CaptureLimits, + stderr: ReaderWorker, + stdout: ReaderWorker, +} + +impl CapturedProcess { + fn start( + program: &'static str, + command: &mut Command, + spawn: impl FnOnce(&mut Command) -> Result, + interrupts: InterruptGuard, + limits: CaptureLimits, + ) -> Result { + command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + let mut child = spawn(command).map_err(|source| ProcessError::Io { + program, + action: "spawn", + source, + })?; + let stdout = child.stdout.take().ok_or(ProcessError::MissingStream { + program, + stream: "stdout", + }); + let stderr = child.stderr.take().ok_or(ProcessError::MissingStream { + program, + stream: "stderr", + }); + let (stdout, stderr) = match (stdout, stderr) { + (Ok(stdout), Ok(stderr)) => (stdout, stderr), + (Err(error), _) | (_, Err(error)) => { + return Err(cleanup_process(&mut child, error)); + } + }; + let stdout = match ReaderWorker::start(program, "stdout", stdout, limits.stdout_bytes()) { + Ok(reader) => reader, + Err(error) => return Err(cleanup_process(&mut child, error)), + }; + let stderr = match ReaderWorker::start(program, "stderr", stderr, limits.stderr_bytes()) { + Ok(reader) => reader, + Err(error) => { + let error = cleanup_process(&mut child, error); + return Err(join_after_cleanup(stdout, error)); + } + }; + Ok(Self { + child, + interrupts, + limits, + stderr, + stdout, + }) + } + + fn finish( + mut self, + program: &'static str, + deadline: &ProcessDeadline, + ) -> Result { + let stdout = match self.stdout.receive(deadline, &self.interrupts) { + Ok(output) => output, + Err(error) => return Err(self.cleanup_readers(error)), + }; + let stderr = match self.stderr.receive(deadline, &self.interrupts) { + Ok(output) => output, + Err(error) => return Err(self.cleanup_readers(error)), + }; + if let Err(error) = self.stdout.join() { + let error = cleanup_process(&mut self.child, error); + return Err(join_after_cleanup(self.stderr, error)); + } + if let Err(error) = self.stderr.join() { + return Err(cleanup_process(&mut self.child, error)); + } + let status = wait_for_child(program, &mut self.child, deadline, &self.interrupts)?; + refuse_exceeded(program, "stdout", self.limits.stdout_bytes(), &stdout).and_then(|()| { + refuse_exceeded(program, "stderr", self.limits.stderr_bytes(), &stderr) + })?; + if let Some(error) = self.interrupts.refusal(program) { + return Err(error); + } + Ok(ProcessOutput { + code: status.code(), + succeeded: status.success(), + stdout: stdout.bytes, + stderr: stderr.bytes, + }) + } + + fn cleanup_readers(self, error: ProcessError) -> ProcessError { + let mut child = self.child; + let error = cleanup_process(&mut child, error); + join_readers(self.stdout, self.stderr, error) + } +} + +pub(super) fn wait_for_child( + program: &'static str, + child: &mut Child, + deadline: &ProcessDeadline, + interrupts: &InterruptGuard, +) -> Result { + loop { + if let Some(error) = interrupts.refusal(program) { + return Err(cleanup_process(child, error)); + } + match child.try_wait() { + Err(source) => { + return Err(cleanup_process( + child, + ProcessError::Io { + program, + action: "poll", + source, + }, + )); + } + Ok(Some(status)) => return Ok(status), + Ok(None) => { + let interval = match deadline.remaining(program) { + Ok(Some((remaining, _duration))) => PROCESS_POLL_INTERVAL.min(remaining), + Ok(None) => PROCESS_POLL_INTERVAL, + Err(error) => return Err(cleanup_process(child, error)), + }; + thread::sleep(interval); + } + } + } +} + +const fn refuse_exceeded( + program: &'static str, + stream: &'static str, + maximum: usize, + output: &BoundedBytes, +) -> Result<(), ProcessError> { + if output.exceeded { + Err(ProcessError::OutputLimit { + program, + stream, + maximum, + }) + } else { + Ok(()) + } +} diff --git a/xtask/src/bounded_process/capture_limit.rs b/xtask/src/bounded_process/capture_limit.rs new file mode 100644 index 0000000..d6040b2 --- /dev/null +++ b/xtask/src/bounded_process/capture_limit.rs @@ -0,0 +1,29 @@ +//! This module owns retained-byte limits for captured child streams. + +/// Maximum retained bytes for one child standard-output and standard-error pair. +#[derive(Clone, Copy)] +pub(crate) struct CaptureLimits { + stderr_bytes: usize, + stdout_bytes: usize, +} + +impl CaptureLimits { + /// Creates exact independent limits for standard output and standard error. + #[must_use] + pub(crate) const fn new(stdout_bytes: usize, stderr_bytes: usize) -> Self { + Self { + stderr_bytes, + stdout_bytes, + } + } + + /// Returns the standard-error retained-byte limit. + pub(super) const fn stderr_bytes(self) -> usize { + self.stderr_bytes + } + + /// Returns the standard-output retained-byte limit. + pub(super) const fn stdout_bytes(self) -> usize { + self.stdout_bytes + } +} diff --git a/xtask/src/bounded_process/cleanup.rs b/xtask/src/bounded_process/cleanup.rs new file mode 100644 index 0000000..df05746 --- /dev/null +++ b/xtask/src/bounded_process/cleanup.rs @@ -0,0 +1,56 @@ +//! This module owns failed child-process and reader teardown. + +use std::io; +use std::process::Child; + +use super::process_group::ProcessGroup; +use super::{ProcessError, ReaderWorker}; + +pub(super) fn cleanup_process(child: &mut Child, primary: ProcessError) -> ProcessError { + let process_group = ProcessGroup::for_child(child).and_then(ProcessGroup::terminate); + let kill = child.kill(); + let wait = child.wait(); + if let Err(source) = process_group { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "terminate child process group", + source, + }; + } + if let Err(source) = kill + && source.kind() != io::ErrorKind::InvalidInput + { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "kill child process", + source, + }; + } + if let Err(source) = wait { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "reap child process", + source, + }; + } + primary +} + +pub(super) fn join_readers( + stdout: ReaderWorker, + stderr: ReaderWorker, + primary: ProcessError, +) -> ProcessError { + let primary = join_after_cleanup(stdout, primary); + join_after_cleanup(stderr, primary) +} + +pub(super) fn join_after_cleanup(reader: ReaderWorker, primary: ProcessError) -> ProcessError { + match reader.join() { + Ok(()) => primary, + Err(additional) => ProcessError::Additional { + primary: Box::new(primary), + additional: Box::new(additional), + }, + } +} diff --git a/xtask/src/bounded_process/deadline.rs b/xtask/src/bounded_process/deadline.rs new file mode 100644 index 0000000..3e12afe --- /dev/null +++ b/xtask/src/bounded_process/deadline.rs @@ -0,0 +1,44 @@ +//! This module owns one deadline spanning process and output collection. + +use std::time::{Duration, Instant}; + +use super::ProcessError; + +pub(super) enum ProcessDeadline { + Bounded { + duration: Duration, + expires: Instant, + }, + Unbounded, +} + +impl ProcessDeadline { + pub(super) fn new( + program: &'static str, + duration: Option, + ) -> Result { + let Some(duration) = duration else { + return Ok(Self::Unbounded); + }; + let expires = Instant::now() + .checked_add(duration) + .ok_or(ProcessError::Timeout { program, duration })?; + Ok(Self::Bounded { duration, expires }) + } + + pub(super) fn remaining( + &self, + program: &'static str, + ) -> Result, ProcessError> { + match self { + Self::Unbounded => Ok(None), + Self::Bounded { duration, expires } => expires + .checked_duration_since(Instant::now()) + .map(|remaining| Some((remaining, *duration))) + .ok_or(ProcessError::Timeout { + program, + duration: *duration, + }), + } + } +} diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs new file mode 100644 index 0000000..cc46e8e --- /dev/null +++ b/xtask/src/bounded_process/error.rs @@ -0,0 +1,129 @@ +//! This module owns typed bounded-process failures. + +use std::error::Error; +use std::fmt; +use std::io; +use std::time::Duration; + +/// A typed failure from synchronous, bounded child-process execution. +pub(crate) enum ProcessError { + /// Reader or cleanup collection found another failure after the primary one. + Additional { + primary: Box, + additional: Box, + }, + /// Process-group termination or child reaping failed after a primary error. + Cleanup { + primary: Box, + action: &'static str, + source: io::Error, + }, + /// A named operating-system process action failed. + Io { + program: &'static str, + action: &'static str, + source: io::Error, + }, + /// A terminal signal interrupted the complete child operation. + Interrupted { + program: &'static str, + signal: &'static str, + }, + /// A child configured for capture did not expose the requested pipe. + MissingStream { + program: &'static str, + stream: &'static str, + }, + /// A captured stream exceeded its fixed retained-byte limit. + OutputLimit { + program: &'static str, + stream: &'static str, + maximum: usize, + }, + /// A dedicated output-reader thread stopped by panicking. + ReaderPanic { + program: &'static str, + stream: &'static str, + }, + /// The complete child operation exceeded its admitted duration. + Timeout { + program: &'static str, + duration: Duration, + }, +} + +impl ProcessError { + /// Reports whether the primary process I/O failure is executable absence. + pub(crate) fn is_not_found(&self) -> bool { + match self { + Self::Additional { primary, .. } | Self::Cleanup { primary, .. } => { + primary.is_not_found() + } + Self::Io { source, .. } => source.kind() == io::ErrorKind::NotFound, + Self::Interrupted { .. } + | Self::MissingStream { .. } + | Self::OutputLimit { .. } + | Self::ReaderPanic { .. } + | Self::Timeout { .. } => false, + } + } +} + +impl fmt::Debug for ProcessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for ProcessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Additional { + primary, + additional, + } => write!(formatter, "{primary}; additionally {additional}"), + Self::Cleanup { + primary, action, .. + } => write!(formatter, "{primary}; additionally failed to {action}"), + Self::Io { + program, action, .. + } => write!(formatter, "cannot {action} {program} process"), + Self::Interrupted { program, signal } => { + write!(formatter, "{program} process was interrupted by {signal}") + } + Self::MissingStream { program, stream } => { + write!(formatter, "{program} {stream} pipe is unavailable") + } + Self::OutputLimit { + program, + stream, + maximum, + } => write!( + formatter, + "{program} {stream} exceeds the {maximum}-byte bound" + ), + Self::ReaderPanic { program, stream } => { + write!(formatter, "{program} {stream} reader panicked") + } + Self::Timeout { program, duration } => write!( + formatter, + "{program} process exceeded its {}-second deadline", + duration.as_secs() + ), + } + } +} + +impl Error for ProcessError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Additional { primary, .. } => Some(primary), + Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), + Self::Interrupted { .. } + | Self::MissingStream { .. } + | Self::OutputLimit { .. } + | Self::ReaderPanic { .. } + | Self::Timeout { .. } => None, + } + } +} diff --git a/xtask/src/bounded_process/interrupt.rs b/xtask/src/bounded_process/interrupt.rs new file mode 100644 index 0000000..a243fd1 --- /dev/null +++ b/xtask/src/bounded_process/interrupt.rs @@ -0,0 +1,224 @@ +//! This module owns terminal-signal refusal for active child groups. + +use std::io; +use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::thread; + +use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM}; +use signal_hook::iterator::Signals; + +use super::ProcessError; + +const HANDLED_SIGNALS: [i32; 4] = [SIGINT, SIGTERM, SIGHUP, SIGQUIT]; + +static CONTROLLER: OnceLock = OnceLock::new(); +static CONTROLLER_START: Mutex<()> = Mutex::new(()); + +/// Registers one active child operation for typed terminal-signal refusal. +/// +/// Dropping an unobserved guard restores the operating system's default +/// handling for the pending signal rather than silently consuming it. +pub(super) struct InterruptGuard { + registry: Arc>>>, + state: Arc, +} + +struct InterruptController { + registry: Arc>>>, +} + +struct InterruptState { + observed: AtomicBool, + signal: AtomicI32, +} + +enum SignalAdmission { + First, + Repeated, +} + +enum DispatchOutcome { + Admitted, + UseDefault, +} + +impl InterruptGuard { + /// Registers `program` with the shared signal controller. + /// + /// Initialization and registry-lock failures remain typed process I/O + /// failures. Registration does not block on signal delivery. + pub(super) fn begin(program: &'static str) -> Result { + interrupt_controller(program)?.register(program) + } + + /// Returns the first terminal signal observed for this operation. + /// + /// Observation is nonblocking. The returned refusal consumes the guard's + /// obligation to restore default handling for that signal on drop. + pub(super) fn refusal(&self, program: &'static str) -> Option { + let signal = self.state.observe()?; + let signal_name = signal_hook::low_level::signal_name(signal).unwrap_or("unknown signal"); + Some(ProcessError::Interrupted { + program, + signal: signal_name, + }) + } +} + +fn interrupt_controller( + program: &'static str, +) -> Result<&'static InterruptController, ProcessError> { + if let Some(controller) = CONTROLLER.get() { + return Ok(controller); + } + let start_guard = CONTROLLER_START.lock().map_err(|_| ProcessError::Io { + program, + action: "serialize terminal signal guard initialization for", + source: io::Error::other("terminal signal initializer is poisoned"), + })?; + if CONTROLLER.get().is_none() { + let controller = InterruptController::start().map_err(|source| ProcessError::Io { + program, + action: "initialize terminal signal guard for", + source, + })?; + CONTROLLER.set(controller).map_err(|_| ProcessError::Io { + program, + action: "publish terminal signal guard for", + source: io::Error::new( + io::ErrorKind::AlreadyExists, + "terminal signal controller was already initialized", + ), + })?; + } + let controller = CONTROLLER.get().ok_or_else(|| ProcessError::Io { + program, + action: "read terminal signal guard for", + source: io::Error::other("terminal signal controller was not initialized"), + })?; + drop(start_guard); + Ok(controller) +} + +impl Drop for InterruptGuard { + fn drop(&mut self) { + if let Ok(mut registry) = self.registry.lock() { + registry.retain(|candidate| { + candidate + .upgrade() + .is_some_and(|state| !Arc::ptr_eq(&state, &self.state)) + }); + } + if let Some(signal) = self.state.unobserved() { + terminate_by_default(signal); + } + } +} + +impl InterruptController { + fn start() -> Result { + let signals = Signals::new(HANDLED_SIGNALS)?; + let registry = Arc::new(Mutex::new(Vec::new())); + let signal_registry = Arc::clone(®istry); + let worker = thread::Builder::new() + .name(String::from("xtask-signal-guard")) + .spawn(move || dispatch(signals, &signal_registry))?; + drop(worker); + Ok(Self { registry }) + } + + fn register(&self, program: &'static str) -> Result { + let state = Arc::new(InterruptState::new()); + self.registry + .lock() + .map_err(|_| ProcessError::Io { + program, + action: "register active child for", + source: io::Error::other("terminal signal registry is poisoned"), + })? + .push(Arc::downgrade(&state)); + Ok(InterruptGuard { + registry: Arc::clone(&self.registry), + state, + }) + } +} + +impl InterruptState { + const fn new() -> Self { + Self { + observed: AtomicBool::new(false), + signal: AtomicI32::new(0), + } + } + + fn interrupt(&self, signal: i32) -> SignalAdmission { + if self + .signal + .compare_exchange(0, signal, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + SignalAdmission::First + } else { + SignalAdmission::Repeated + } + } + + fn observe(&self) -> Option { + let signal = self.signal.load(Ordering::Acquire); + if signal == 0 { + return None; + } + self.observed.store(true, Ordering::Release); + Some(signal) + } + + fn unobserved(&self) -> Option { + let signal = self.signal.load(Ordering::Acquire); + (signal != 0 && !self.observed.load(Ordering::Acquire)).then_some(signal) + } +} + +fn dispatch(mut signals: Signals, registry: &Mutex>>) { + for signal in signals.forever() { + if !matches!( + dispatch_signal(registry, signal), + Ok(DispatchOutcome::Admitted) + ) { + terminate_by_default(signal); + } + } +} + +fn dispatch_signal( + registry: &Mutex>>, + signal: i32, +) -> Result { + let mut registry = registry.lock().map_err(|_| ())?; + let mut has_active_operation = false; + let mut repeated = false; + registry.retain(|candidate| { + candidate.upgrade().is_some_and(|state| { + has_active_operation = true; + repeated |= matches!(state.interrupt(signal), SignalAdmission::Repeated); + true + }) + }); + drop(registry); + if !has_active_operation || repeated { + Ok(DispatchOutcome::UseDefault) + } else { + Ok(DispatchOutcome::Admitted) + } +} + +fn terminate_by_default(signal: i32) { + if signal_hook::low_level::emulate_default_handler(signal).is_err() { + signal_hook::low_level::abort(); + } +} + +#[cfg(test)] +#[path = "interrupt/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/interrupt/tests.rs b/xtask/src/bounded_process/interrupt/tests.rs new file mode 100644 index 0000000..a97ee35 --- /dev/null +++ b/xtask/src/bounded_process/interrupt/tests.rs @@ -0,0 +1,34 @@ +//! This module owns terminal-signal guard state-transition laws. + +use signal_hook::consts::{SIGINT, SIGTERM}; + +use super::{InterruptState, SignalAdmission}; + +#[test] +fn first_terminal_signal_becomes_one_observed_refusal() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGINT), SignalAdmission::First)); + assert_eq!(state.observe(), Some(SIGINT)); + assert_eq!(state.unobserved(), None); +} + +#[test] +fn retirement_detects_an_unobserved_terminal_signal() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGTERM), SignalAdmission::First)); + assert_eq!(state.unobserved(), Some(SIGTERM)); +} + +#[test] +fn a_second_terminal_signal_requires_default_termination() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGINT), SignalAdmission::First)); + assert!(matches!( + state.interrupt(SIGTERM), + SignalAdmission::Repeated + )); + assert_eq!(state.observe(), Some(SIGINT)); +} diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs new file mode 100644 index 0000000..78f264d --- /dev/null +++ b/xtask/src/bounded_process/process_group.rs @@ -0,0 +1,104 @@ +//! This module owns child process-group creation and termination. + +use std::io; +use std::process::Child; + +#[cfg(test)] +use std::io::Read; +#[cfg(test)] +use std::os::unix::net::UnixListener; +#[cfg(test)] +use std::path::Path; + +use rustix::io::Errno; +use rustix::process::{Pid, Signal, kill_process_group}; + +/// The dedicated process-group identity established for one spawned child. +pub(super) struct ProcessGroup(Pid); + +impl ProcessGroup { + /// Admits the child's nonzero operating-system identifier as a group ID. + /// + /// Conversion fails when the unsigned child ID does not fit the platform's + /// signed PID representation or when the observed ID is zero. + pub(super) fn for_child(child: &Child) -> Result { + let raw = i32::try_from(child.id()) + .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source))?; + let pid = Pid::from_raw(raw) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "child PID is zero"))?; + Ok(Self(pid)) + } + + /// Sends `SIGKILL` to every member of the admitted process group. + /// + /// An absent group is already terminated and succeeds. Other operating + /// system errors are preserved. + pub(super) fn terminate(self) -> Result<(), io::Error> { + match kill_process_group(self.0, Signal::KILL) { + Ok(()) | Err(Errno::SRCH) => Ok(()), + Err(source) => Err(source.into()), + } + } +} + +#[cfg(test)] +const DESCENDANT_CHILD: &str = "KEEP_XTASK_DESCENDANT_CHILD"; +#[cfg(test)] +const DESCENDANT_CHILD_READY: &str = "KEEP_XTASK_DESCENDANT_CHILD_READY"; +#[cfg(test)] +const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; +#[cfg(test)] +const DESCENDANT_READY: &str = "KEEP_XTASK_DESCENDANT_READY"; +#[cfg(test)] +const DESCENDANT_SOCKET: &str = "KEEP_XTASK_DESCENDANT_SOCKET"; +#[cfg(test)] +const INTERRUPT_SUPERVISOR: &str = "KEEP_XTASK_INTERRUPT_SUPERVISOR"; + +#[cfg(test)] +fn readiness_listener(path: &Path) -> Result { + let listener = UnixListener::bind(path)?; + listener.set_nonblocking(true)?; + Ok(listener) +} + +#[cfg(test)] +fn wait_for_ready(listener: &UnixListener, child: &mut Child) -> Result<(), io::Error> { + loop { + match listener.accept() { + Ok((mut stream, _address)) => { + stream.set_nonblocking(false)?; + let mut signal = [0_u8; 1]; + stream.read_exact(&mut signal)?; + if signal != [b'r'] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "descendant sent an invalid readiness signal", + )); + } + return Ok(()); + } + Err(source) if source.kind() == io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait()? { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("child exited before descendant readiness: {status}"), + )); + } + std::thread::yield_now(); + } + Err(source) => return Err(source), + } + } +} + +#[cfg(test)] +#[path = "process_group/child_tests.rs"] +mod child_tests; + +#[cfg(test)] +#[path = "process_group/readiness_tests.rs"] +mod readiness_tests; + +#[cfg(test)] +#[path = "process_group/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/process_group/child_tests.rs b/xtask/src/bounded_process/process_group/child_tests.rs new file mode 100644 index 0000000..f2adb43 --- /dev/null +++ b/xtask/src/bounded_process/process_group/child_tests.rs @@ -0,0 +1,126 @@ +//! This module owns subprocess fixtures for process-group regression tests. + +use std::env; +use std::io::{self, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::process::Command; + +use super::{ + DESCENDANT_CHILD, DESCENDANT_CHILD_READY, DESCENDANT_PARENT, DESCENDANT_READY, + DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, readiness_listener, wait_for_ready, +}; +use crate::bounded_process::{ProcessError, capture}; + +#[test] +fn process_supervisor_captures_descendant_until_interrupted() -> Result<(), io::Error> { + if env::var_os(INTERRUPT_SUPERVISOR).is_none() { + return Ok(()); + } + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open", + ]) + .env(DESCENDANT_PARENT, "1") + .env( + DESCENDANT_CHILD_READY, + env::var_os(DESCENDANT_CHILD_READY).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant child-ready path", + ) + })?, + ) + .env( + DESCENDANT_READY, + env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?, + ) + .env( + DESCENDANT_SOCKET, + env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant socket path") + })?, + ); + match capture("interrupt fixture", &mut command, None) { + Err(ProcessError::Interrupted { + program: "interrupt fixture", + signal: "SIGINT", + }) => Ok(()), + Err(error) => Err(io::Error::other(format!( + "unexpected interrupt refusal: {error}" + ))), + Ok(_) => Err(io::Error::other( + "interrupt fixture unexpectedly completed successfully", + )), + } +} + +#[test] +fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { + if env::var_os(DESCENDANT_PARENT).is_none() { + return Ok(()); + } + let executable = env::current_exe()?; + let ready = env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?; + let child_ready = env::var_os(DESCENDANT_CHILD_READY).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant child-ready path", + ) + })?; + let listener = readiness_listener(std::path::Path::new(&child_ready))?; + let mut descendant = Command::new(executable) + .args([ + "--exact", + "bounded_process::process_group::child_tests::process_descendant_holds_pipe", + ]) + .env(DESCENDANT_CHILD, "1") + .env(DESCENDANT_READY, &child_ready) + .env( + DESCENDANT_SOCKET, + env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant socket path", + ) + })?, + ) + .spawn()?; + wait_for_ready(&listener, &mut descendant)?; + let mut readiness = UnixStream::connect(ready)?; + readiness.write_all(b"r")?; + Ok(()) +} + +#[test] +fn process_descendant_holds_pipe() -> Result<(), io::Error> { + if env::var_os(DESCENDANT_CHILD).is_none() { + return Ok(()); + } + let socket = env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant socket path", + ) + })?; + let ready = env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?; + let listener = UnixListener::bind(socket)?; + let mut readiness = UnixStream::connect(ready)?; + readiness.write_all(b"r")?; + loop { + let (mut stream, _) = listener.accept()?; + let mut command = [0_u8; 1]; + io::Read::read_exact(&mut stream, &mut command)?; + if command == [b'x'] { + return Ok(()); + } + } +} diff --git a/xtask/src/bounded_process/process_group/readiness_tests.rs b/xtask/src/bounded_process/process_group/readiness_tests.rs new file mode 100644 index 0000000..207bbf5 --- /dev/null +++ b/xtask/src/bounded_process/process_group/readiness_tests.rs @@ -0,0 +1,42 @@ +//! This module owns deterministic readiness synchronization evidence. + +use std::env; +use std::io; +use std::process::{Command, Stdio}; + +use super::{readiness_listener, wait_for_ready}; +use crate::test_directory::TestDirectory; + +#[test] +fn readiness_wait_does_not_depend_on_wall_clock() { + let process_group = include_str!("../process_group.rs"); + + assert!( + !process_group.contains("std::time::"), + "readiness must be bounded by the child lifecycle, not wall-clock time" + ); +} + +#[test] +fn readiness_refuses_child_exit_without_a_signal() -> Result<(), Box> { + let directory = TestDirectory::create("readiness-exit")?; + let listener = readiness_listener(&directory.path().join("ready"))?; + let mut child = Command::new(env::current_exe()?) + .args([ + "--exact", + "bounded_process::process_group::readiness_tests::readiness_wait_does_not_depend_on_wall_clock", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + let result = wait_for_ready(&listener, &mut child); + directory.close()?; + + assert!(matches!( + result, + Err(source) if source.kind() == io::ErrorKind::UnexpectedEof + )); + Ok(()) +} diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs new file mode 100644 index 0000000..1e5017a --- /dev/null +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -0,0 +1,222 @@ +//! This module owns child process-group cleanup regression evidence. + +use std::env; +use std::io; +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::path::Path; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use rustix::process::{Pid, Signal, kill_process}; + +use super::{ + DESCENDANT_CHILD_READY, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, + INTERRUPT_SUPERVISOR, readiness_listener, wait_for_ready, +}; +use crate::bounded_process::cleanup::cleanup_process; +use crate::bounded_process::{ProcessError, capture_with}; +use crate::test_directory::TestDirectory; + +const CHILD_PROCESS: &str = + "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open"; +const SUPERVISOR_PROCESS: &str = "bounded_process::process_group::child_tests::process_supervisor_captures_descendant_until_interrupted"; + +fn spawn_ready_descendant( + executable: &Path, + ready: &Path, + child_ready: &Path, + socket: &Path, +) -> Result> { + let listener = readiness_listener(ready)?; + let mut child = Command::new(executable) + .args(["--exact", CHILD_PROCESS]) + .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_CHILD_READY, child_ready) + .env(DESCENDANT_READY, ready) + .env(DESCENDANT_SOCKET, socket) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn()?; + if let Err(source) = wait_for_ready(&listener, &mut child) { + let error = cleanup_process( + &mut child, + ProcessError::Io { + program: "test process", + action: "wait for descendant readiness in", + source, + }, + ); + return Err(Box::new(error)); + } + Ok(child) +} + +#[test] +fn terminal_interrupt_terminates_the_isolated_descendant_group() +-> Result<(), Box> { + let directory = TestDirectory::create("process-group-interrupt")?; + let ready = directory.path().join("ready"); + let child_ready = directory.path().join("child-ready"); + let socket = directory.path().join("descendant.sock"); + let listener = readiness_listener(&ready)?; + let executable = env::current_exe()?; + let mut supervisor = Command::new(executable) + .args(["--exact", SUPERVISOR_PROCESS]) + .env(INTERRUPT_SUPERVISOR, "1") + .env(DESCENDANT_CHILD_READY, &child_ready) + .env(DESCENDANT_READY, &ready) + .env(DESCENDANT_SOCKET, &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + wait_for_ready(&listener, &mut supervisor)?; + let descendant = UnixStream::connect(&socket)?; + let supervisor_pid = + Pid::from_raw(i32::try_from(supervisor.id())?).ok_or("supervisor process ID is zero")?; + kill_process(supervisor_pid, Signal::INT)?; + + let status = wait_for_exit(&mut supervisor)?; + require_descendant_disconnect(descendant)?; + directory.close()?; + assert!( + status.success(), + "interrupted supervisor did not exit cleanly: {status:?}" + ); + Ok(()) +} + +#[test] +fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> +{ + let directory = TestDirectory::create("pd")?; + let ready = directory.path().join("r"); + let child_ready = directory.path().join("cr"); + let socket = directory.path().join("d"); + let executable = env::current_exe()?; + let child = spawn_ready_descendant(&executable, &ready, &child_ready, &socket)?; + + let mut admitted = Command::new("pre-spawned descendant fixture"); + let result = capture_with( + "test process", + &mut admitted, + Some(Duration::from_millis(25)), + |_command| Ok(child), + ); + + assert!(matches!( + result, + Err(ProcessError::Timeout { + program: "test process", + duration, + }) if duration == Duration::from_millis(25) + )); + directory.close()?; + Ok(()) +} + +#[test] +fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box> { + let directory = TestDirectory::create("process-group-cleanup")?; + let ready = directory.path().join("ready"); + let child_ready = directory.path().join("child-ready"); + let socket = directory.path().join("descendant.sock"); + let listener = readiness_listener(&ready)?; + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args(["--exact", CHILD_PROCESS]) + .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_CHILD_READY, &child_ready) + .env(DESCENDANT_READY, &ready) + .env(DESCENDANT_SOCKET, &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0); + let mut child = command.spawn()?; + wait_for_ready(&listener, &mut child)?; + let descendant = UnixStream::connect(&socket)?; + + let error = cleanup_process( + &mut child, + ProcessError::Timeout { + program: "test process", + duration: Duration::from_millis(25), + }, + ); + + assert!(matches!(error, ProcessError::Timeout { .. })); + require_descendant_disconnect(descendant)?; + directory.close()?; + Ok(()) +} + +fn require_descendant_disconnect(mut descendant: UnixStream) -> Result<(), io::Error> { + match descendant.set_read_timeout(Some(Duration::from_millis(250))) { + Err(source) if source.kind() == io::ErrorKind::InvalidInput => { + descendant.set_nonblocking(true)?; + } + result => result?, + } + let mut byte = [0_u8; 1]; + match io::Read::read_exact(&mut descendant, &mut byte) { + Err(source) + if matches!( + source.kind(), + io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset + ) => + { + Ok(()) + } + Err(source) => Err(source), + Ok(()) => Err(io::Error::other( + "terminated descendant unexpectedly wrote to its witness socket", + )), + } +} + +#[test] +fn descendant_disconnect_witness_has_a_finite_read_deadline() -> Result<(), io::Error> { + let (descendant, _retained_writer) = UnixStream::pair()?; + + let Err(error) = require_descendant_disconnect(descendant) else { + return Err(io::Error::other( + "an open idle witness did not reach its read deadline", + )); + }; + + assert!(matches!( + error.kind(), + io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock + )); + Ok(()) +} + +#[test] +fn descendant_disconnect_witness_accepts_an_already_closed_peer() -> Result<(), io::Error> { + let (descendant, peer) = UnixStream::pair()?; + drop(peer); + + require_descendant_disconnect(descendant) +} + +fn wait_for_exit(child: &mut std::process::Child) -> Result { + let expires = Instant::now() + .checked_add(Duration::from_secs(2)) + .ok_or_else(|| io::Error::other("supervisor exit deadline overflow"))?; + loop { + if let Some(status) = child.try_wait()? { + return Ok(status); + } + if Instant::now() >= expires { + child.kill()?; + return child.wait(); + } + thread::yield_now(); + } +} diff --git a/xtask/src/bounded_process/reader.rs b/xtask/src/bounded_process/reader.rs new file mode 100644 index 0000000..c1f423d --- /dev/null +++ b/xtask/src/bounded_process/reader.rs @@ -0,0 +1,104 @@ +//! This module owns deadline-bounded child-output collection. + +use std::io; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::thread::{self, JoinHandle}; + +use crate::process_output::{BoundedBytes, bounded_bytes}; + +use super::{InterruptGuard, ProcessDeadline, ProcessError}; + +const INTERRUPT_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); + +/// Owns one bounded stream reader and its single-result channel. +pub(super) struct ReaderWorker { + handle: JoinHandle<()>, + program: &'static str, + receiver: Receiver>, + stream: &'static str, +} + +impl ReaderWorker { + /// Starts a named reader that retains at most `maximum` stream bytes. + /// + /// Thread creation failure remains a typed process I/O error. The worker + /// sends exactly one bounded result and performs no unbounded buffering. + pub(super) fn start( + program: &'static str, + stream: &'static str, + reader: impl io::Read + Send + 'static, + maximum: usize, + ) -> Result { + let (sender, receiver) = mpsc::sync_channel(1); + let handle = thread::Builder::new() + .name(format!("xtask-{stream}-reader")) + .spawn(move || { + drop(sender.send(bounded_bytes(reader, maximum))); + }) + .map_err(|source| ProcessError::Io { + program, + action: "start output reader", + source, + })?; + Ok(Self { + handle, + program, + receiver, + stream, + }) + } + + /// Waits for the bounded result while polling the shared deadline and signal guard. + /// + /// This call blocks only for the smaller of the remaining deadline and the + /// fixed interrupt interval, so timeout and interruption remain observable. + pub(super) fn receive( + &self, + deadline: &ProcessDeadline, + interrupts: &InterruptGuard, + ) -> Result { + loop { + if let Some(error) = interrupts.refusal(self.program) { + return Err(error); + } + let (wait, duration) = receive_wait(deadline, self.program)?; + match self.receiver.recv_timeout(wait) { + Ok(result) => { + return result.map_err(|source| ProcessError::Io { + program: self.program, + action: "read child output", + source, + }); + } + Err(RecvTimeoutError::Timeout) if duration.is_none() => {} + Err(RecvTimeoutError::Timeout) => { + deadline.remaining(self.program)?; + } + Err(RecvTimeoutError::Disconnected) => { + return Err(reader_panic(self.program, self.stream)); + } + } + } + } + + /// Joins the completed reader and maps a worker panic to a typed failure. + pub(super) fn join(self) -> Result<(), ProcessError> { + self.handle + .join() + .map_err(|_panic| reader_panic(self.program, self.stream)) + } +} + +fn receive_wait( + deadline: &ProcessDeadline, + program: &'static str, +) -> Result<(std::time::Duration, Option), ProcessError> { + match deadline.remaining(program)? { + Some((remaining, duration)) => Ok((remaining.min(INTERRUPT_POLL_INTERVAL), Some(duration))), + None => Ok((INTERRUPT_POLL_INTERVAL, None)), + } +} + +const fn reader_panic(program: &'static str, stream: &'static str) -> ProcessError { + ProcessError::ReaderPanic { program, stream } +} diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs new file mode 100644 index 0000000..5913183 --- /dev/null +++ b/xtask/src/bounded_process/tests.rs @@ -0,0 +1,129 @@ +//! This module owns bounded child-process regression evidence. + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::str; +use std::time::Duration; + +use super::{ProcessError, capture, status}; +use crate::test_directory::TestDirectory; + +const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_CHILD"; + +#[test] +fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box> +{ + let repository = TestDirectory::create("bounded-process-output")?; + let blob = repository.path().join("oversized.bin"); + let global_config = repository.path().join("global.gitconfig"); + let template = repository.path().join("empty-template"); + fs::write(&blob, vec![b'x'; 1_048_577])?; + fs::write(&global_config, [])?; + fs::create_dir(&template)?; + let initialized = fixture_git_command(&repository)? + .args(["init", "--quiet"]) + .arg(template_argument(&template)) + .status()?; + if !initialized.success() { + return Err(io::Error::other("cannot initialize fixture repository").into()); + } + let hashed = fixture_git_command(&repository)? + .args(["hash-object", "-w", "oversized.bin"]) + .output()?; + if !hashed.status.success() { + return Err(io::Error::other("cannot hash fixture blob").into()); + } + let object_id = str::from_utf8(&hashed.stdout)?.trim(); + let mut command = fixture_git_command(&repository)?; + command + .args(["cat-file", "blob", object_id]) + .stdin(Stdio::null()); + + let result = capture("test process", &mut command, Some(Duration::from_secs(5))); + repository.close()?; + + assert!(matches!( + result, + Err(ProcessError::OutputLimit { + program: "test process", + stream: "stdout", + maximum: 1_048_576, + }) + )); + Ok(()) +} + +fn fixture_git_command(repository: &TestDirectory) -> Result { + let path = env::var_os("PATH").ok_or_else(|| io::Error::other("PATH is unavailable"))?; + let global_config = repository.path().join("global.gitconfig"); + let mut command = Command::new("git"); + command + .env_clear() + .env("PATH", path) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", global_config) + .env("LC_ALL", "C") + .current_dir(repository.path()); + Ok(command) +} + +fn template_argument(template: &Path) -> OsString { + let mut argument = OsString::from("--template="); + argument.push(template); + argument +} + +#[test] +fn fixture_template_argument_preserves_non_utf8_paths() { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let template = OsString::from_vec(b"/tmp/non-utf8-\xff".to_vec()); + let argument = template_argument(Path::new(&template)); + + assert_eq!( + argument.as_os_str().as_bytes(), + b"--template=/tmp/non-utf8-\xff" + ); +} + +#[test] +fn inherited_process_obeys_the_process_deadline() -> Result<(), Box> { + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::tests::process_child_parks_indefinitely", + ]) + .env(PARKED_CHILD, "1") + .stdin(Stdio::null()); + + let result = status( + "test process", + &mut command, + Some(Duration::from_millis(50)), + ); + + assert!(matches!( + result, + Err(ProcessError::Timeout { + program: "test process", + duration, + }) if duration == Duration::from_millis(50) + )); + Ok(()) +} + +#[test] +fn process_child_parks_indefinitely() { + if env::var_os(PARKED_CHILD).is_none() { + return; + } + loop { + std::thread::park(); + } +} diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs new file mode 100644 index 0000000..470e7ba --- /dev/null +++ b/xtask/src/documentation_integrity.rs @@ -0,0 +1,77 @@ +//! This module owns documentation and workflow integrity orchestration. + +mod contributor_contract; +mod corpus; +mod dependabot; +mod error; +mod execution; +mod node_toolchain; +mod policy_corpus; +mod repository_text; +mod tool; +mod workflow_contract; + +use std::path::{Path, PathBuf}; + +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +pub(super) use error::DocumentationError; + +/// Runs the pinned-tool malformed-input refusal evidence. +pub(super) fn check_refusals() -> Result<(), DocumentationError> { + execution::check_refusals() +} + +pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { + check_with(repository_path, execution::run) +} + +fn check_with( + repository_path: &Path, + run_tools: impl FnOnce( + &RepositoryProcessDirectory, + &RepositoryRoot, + &corpus::SourceCorpus, + &corpus::SourceCorpus, + ) -> Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { + let repository_root = RepositoryRoot::open(repository_path).map_err(|source| { + DocumentationError::RepositoryRootInspect { + path: repository_path.to_owned(), + source, + } + })?; + let process_directory = repository_root.process_directory().map_err(|source| { + DocumentationError::RepositoryRootInspect { + path: repository_path.to_owned(), + source, + } + })?; + verify_root(&repository_root, repository_path)?; + let policies = policy_corpus::FixedPolicyCorpus::admit(&repository_root, &process_directory)?; + let markdown = corpus::SourceCorpus::markdown(&repository_root, &process_directory)?; + let workflows = corpus::SourceCorpus::workflow(&repository_root, &process_directory)?; + run_tools(&process_directory, &repository_root, &markdown, &workflows)?; + policies.verify(&repository_root)?; + verify_root(&repository_root, repository_path) +} + +fn verify_root( + repository_root: &RepositoryRoot, + repository_path: &Path, +) -> Result<(), DocumentationError> { + match repository_root.is_current_path() { + Ok(true) => Ok(()), + Ok(false) => Err(DocumentationError::RepositoryRootChanged( + repository_path.to_owned(), + )), + Err(source) => Err(DocumentationError::RepositoryRootInspect { + path: PathBuf::from(repository_path), + source, + }), + } +} + +#[cfg(test)] +#[path = "documentation_integrity/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/contributor_contract.rs b/xtask/src/documentation_integrity/contributor_contract.rs new file mode 100644 index 0000000..76017fd --- /dev/null +++ b/xtask/src/documentation_integrity/contributor_contract.rs @@ -0,0 +1,103 @@ +//! This module owns contributor-facing documentation command contracts. + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text::{self, RepositoryText}; + +const CONTRIBUTING_PATH: &str = "CONTRIBUTING.md"; +const STANDARDS_PATH: &str = "docs/Documentation Standards.md"; +const UNSTAGED_CHECK: &str = "git diff --check"; +const STAGED_CHECK: &str = "git diff --cached --check"; +const WHOLE_TREE_CHECK: &str = r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#; + +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result<[RepositoryText; 2], DocumentationError> { + Ok([ + checked_source(repository_root, CONTRIBUTING_PATH)?, + checked_source(repository_root, STANDARDS_PATH)?, + ]) +} + +fn checked_source( + repository_root: &RepositoryRoot, + path: &'static str, +) -> Result { + let raw = repository_text::read(repository_root, path)?; + admit(path, raw.as_str())?; + raw.verify(repository_root)?; + Ok(raw) +} + +fn admit(path: &'static str, raw: &str) -> Result<(), DocumentationError> { + if raw.lines().any(|line| line == WHOLE_TREE_CHECK) { + return Err(contract( + path, + "contributor command does not replace change checks with a whole-tree check", + )); + } + require_line( + path, + raw, + UNSTAGED_CHECK, + "documents the unstaged whitespace check", + )?; + require_line( + path, + raw, + STAGED_CHECK, + "documents the staged whitespace check", + ) +} + +fn require_line( + path: &'static str, + raw: &str, + expected: &str, + requirement: &'static str, +) -> Result<(), DocumentationError> { + if raw.lines().any(|line| line == expected) { + Ok(()) + } else { + Err(contract(path, requirement)) + } +} + +const fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { path, requirement } +} + +#[cfg(test)] +mod tests { + #[test] + fn whole_tree_whitespace_replacement_is_refused() { + let invalid = concat!( + "git diff --check \"$(git hash-object -t tree /dev/null)\" HEAD\n", + "git diff --check\n", + "git diff --cached --check\n", + ); + + assert!(matches!( + super::admit("guide.md", invalid), + Err(super::DocumentationError::RepositoryContract { + path: "guide.md", + requirement: "contributor command does not replace change checks with a whole-tree check", + }) + )); + } + + #[test] + fn contributor_commands_cover_staged_and_unstaged_whitespace() { + for invalid in ["git diff --check\n", "git diff --cached --check\n"] { + assert!(super::admit("guide.md", invalid).is_err()); + } + } + + #[test] + fn separate_change_checks_are_admitted() { + let valid = "git diff --check\ngit diff --cached --check\n"; + + assert!(super::admit("guide.md", valid).is_ok()); + } +} diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs new file mode 100644 index 0000000..df3bbb3 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus.rs @@ -0,0 +1,212 @@ +//! This module owns deterministic documentation source selection. + +mod byte_budget; +mod selection; +mod source_witness; + +use std::cmp::Ordering; +use std::io; +use std::path::Path; + +use xtask::protocol_admission::posix_relative_path; + +use super::error::DocumentationError; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryProcessDirectory, RepositoryRoot}; +use byte_budget::CorpusByteBudget; +#[cfg(test)] +use byte_budget::{CORPUS_FILE_MAX_BYTES, CORPUS_MAX_BYTES}; +use selection::CorpusKind; +use source_witness::AdmittedSource; + +pub(super) struct SourceCorpus { + kind: CorpusKind, + paths: Vec, + sources: Vec, +} + +impl SourceCorpus { + pub(super) fn markdown( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + Self::read(repository_root, process_directory, CorpusKind::Markdown) + } + + pub(super) fn workflow( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + Self::read(repository_root, process_directory, CorpusKind::Workflow) + } + + pub(super) fn paths(&self) -> &[String] { + &self.paths + } + + /// Copies every admitted source into its repository-relative snapshot path. + /// + /// Each copy streams from the retained source descriptor, remains within + /// the corpus byte bounds admitted during construction, and revalidates the + /// source identity around the copy. Filesystem or identity failures are + /// returned through [`DocumentationError`]. + pub(super) fn materialize(&self, snapshot_root: &Path) -> Result<(), DocumentationError> { + for source in &self.sources { + source.materialize(snapshot_root, self.kind)?; + } + Ok(()) + } + + /// Revalidates every admitted source identity and the exact corpus membership. + /// + /// Each retained source must still match the device, inode, byte length, + /// modification time, and change time admitted for this corpus kind. The + /// method then reruns the bounded Git present/deleted inventories for the + /// same kind, requires the sorted selected path set to remain identical, + /// and revalidates every retained identity after re-inventory. + /// + /// This operation performs repository metadata I/O and starts bounded Git + /// child processes. It reports path replacement, in-place mutation, added + /// or removed membership, unsafe or non-UTF-8 paths, nonregular sources, and + /// Git or filesystem failures through [`DocumentationError`]. Successful + /// completion confirms that every source and the corpus set remain + /// unchanged; it does not reread source contents. + pub(super) fn verify_unchanged( + &self, + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result<(), DocumentationError> { + self.verify_unchanged_with(repository_root, process_directory, || Ok(())) + } + + fn verify_unchanged_with( + &self, + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + after_identity_verification: impl FnOnce() -> Result<(), DocumentationError>, + ) -> Result<(), DocumentationError> { + self.verify_sources(repository_root)?; + after_identity_verification()?; + let current = Self::read(repository_root, process_directory, self.kind)?; + if let Some(path) = membership_change(&self.paths, ¤t.paths) { + return Err(DocumentationError::CorpusChanged { + corpus: self.kind.label(), + path, + }); + } + self.verify_sources(repository_root) + } + + fn verify_sources(&self, repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { + for source in &self.sources { + source.verify(repository_root, self.kind)?; + } + Ok(()) + } + + fn read( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + kind: CorpusKind, + ) -> Result { + let present = paths_with( + kind.present_arguments(), + kind.present_operation(), + |command| process_directory.spawn(command), + )?; + let deleted = paths_with( + kind.deleted_arguments(), + kind.deleted_operation(), + |command| process_directory.spawn(command), + )?; + let selected = present.difference(&deleted); + let sources = admit_paths(repository_root, selected, kind)?; + if sources.is_empty() { + Err(DocumentationError::EmptyCorpus(kind.label())) + } else { + let paths = sources + .iter() + .map(|source| source.path().to_owned()) + .collect(); + Ok(Self { + kind, + paths, + sources, + }) + } + } +} + +fn membership_change(expected: &[String], observed: &[String]) -> Option { + let mut expected = expected.iter().peekable(); + let mut observed = observed.iter().peekable(); + loop { + match (expected.peek(), observed.peek()) { + (Some(left), Some(right)) => match left.cmp(right) { + Ordering::Equal => { + expected.next(); + observed.next(); + } + Ordering::Less => return Some((*left).clone()), + Ordering::Greater => return Some((*right).clone()), + }, + (Some(path), None) | (None, Some(path)) => return Some((*path).clone()), + (None, None) => return None, + } + } +} + +fn admit_paths<'a>( + repository_root: &RepositoryRoot, + paths: impl Iterator, + kind: CorpusKind, +) -> Result, DocumentationError> { + let mut admitted = Vec::new(); + let mut budget = CorpusByteBudget::default(); + for path in paths { + if let Some(source) = admit_path(repository_root, path, kind)? { + budget.admit(kind, source.path(), source.bytes())?; + admitted.push(source); + } + } + Ok(admitted) +} + +fn admit_path( + repository_root: &RepositoryRoot, + path: &GitPath, + kind: CorpusKind, +) -> Result, DocumentationError> { + let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { + DocumentationError::PathEncoding { + corpus: kind.label(), + source, + } + })?; + let relative = posix_relative_path(&text).map_err(|_| DocumentationError::InvalidPath { + corpus: kind.label(), + path: text.clone(), + })?; + match repository_root.open_file(&relative) { + Ok(file) => Ok(Some(AdmittedSource::admit(file, text, relative, kind)?)), + Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { + Ok(None) + } + Err(OpenRepositoryFileError::Io(source)) => Err(DocumentationError::Inspect { + corpus: kind.label(), + path: text, + source, + }), + Err(OpenRepositoryFileError::NonRegular) => Err(DocumentationError::NonRegular { + corpus: kind.label(), + path: text, + }), + } +} + +#[cfg(test)] +#[path = "corpus/replacement_tests.rs"] +mod replacement_tests; +#[cfg(test)] +#[path = "corpus/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/corpus/byte_budget.rs b/xtask/src/documentation_integrity/corpus/byte_budget.rs new file mode 100644 index 0000000..405175c --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/byte_budget.rs @@ -0,0 +1,47 @@ +//! This module owns documentation corpus byte-budget admission. + +use super::CorpusKind; +use crate::documentation_integrity::error::DocumentationError; + +/// Maximum admitted bytes for one documentation source. +pub(super) const CORPUS_FILE_MAX_BYTES: u64 = 4_194_304; +/// Maximum admitted bytes for one complete documentation corpus. +pub(super) const CORPUS_MAX_BYTES: u64 = 67_108_864; + +/// Checked aggregate byte accounting for one corpus. +#[derive(Default)] +pub(super) struct CorpusByteBudget { + observed: u64, +} + +impl CorpusByteBudget { + /// Admits one bounded source and advances the aggregate count. + pub(super) fn admit( + &mut self, + kind: CorpusKind, + path: &str, + bytes: u64, + ) -> Result<(), DocumentationError> { + if bytes > CORPUS_FILE_MAX_BYTES { + return Err(DocumentationError::CorpusFileTooLarge { + corpus: kind.label(), + path: path.to_owned(), + maximum: CORPUS_FILE_MAX_BYTES, + observed: bytes, + }); + } + let observed = self + .observed + .checked_add(bytes) + .ok_or_else(|| DocumentationError::CorpusSizeOverflow(kind.label()))?; + if observed > CORPUS_MAX_BYTES { + return Err(DocumentationError::CorpusTooLarge { + corpus: kind.label(), + maximum: CORPUS_MAX_BYTES, + observed, + }); + } + self.observed = observed; + Ok(()) + } +} diff --git a/xtask/src/documentation_integrity/corpus/replacement_tests.rs b/xtask/src/documentation_integrity/corpus/replacement_tests.rs new file mode 100644 index 0000000..53bd07a --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/replacement_tests.rs @@ -0,0 +1,96 @@ +//! This module owns documentation source-replacement regression evidence. + +use std::fs; + +use super::SourceCorpus; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +#[test] +fn selected_source_replacement_refuses_the_admitted_corpus() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-source-replacement")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + fs::rename(root.join("selected.md"), root.join("retained.md"))?; + fs::write(root.join("selected.md"), "# Substitute\n")?; + + let result = corpus.verify_unchanged(&repository_root, &process_directory); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "selected.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} + +#[test] +fn added_source_refuses_the_admitted_corpus() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-source-addition")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + fs::write(root.join("added.md"), "# Added\n")?; + + let result = corpus.verify_unchanged(&repository_root, &process_directory); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "added.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} + +#[test] +fn source_replaced_between_identity_and_membership_checks_is_refused() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-source-verification-race")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + fs::write(root.join("substitute.bin"), "# Substitute\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + let result = corpus.verify_unchanged_with(&repository_root, &process_directory, || { + fs::rename(root.join("substitute.bin"), root.join("selected.md")).map_err(|source| { + DocumentationError::Snapshot { + action: "replace source between corpus verification phases", + source, + } + }) + }); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "selected.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/corpus/selection.rs b/xtask/src/documentation_integrity/corpus/selection.rs new file mode 100644 index 0000000..98d1e36 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/selection.rs @@ -0,0 +1,73 @@ +//! This module owns Git path-selection policy for documentation corpora. + +const MARKDOWN_PRESENT: [&str; 7] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + "*.md", +]; +const MARKDOWN_DELETED: [&str; 5] = ["ls-files", "-z", "--deleted", "--", "*.md"]; +const WORKFLOW_PRESENT: [&str; 8] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; +const WORKFLOW_DELETED: [&str; 6] = [ + "ls-files", + "-z", + "--deleted", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; + +#[derive(Clone, Copy)] +pub(super) enum CorpusKind { + Markdown, + Workflow, +} + +impl CorpusKind { + pub(super) const fn label(self) -> &'static str { + match self { + Self::Markdown => "Markdown", + Self::Workflow => "GitHub Actions workflow", + } + } + + pub(super) const fn present_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_PRESENT, + Self::Workflow => &WORKFLOW_PRESENT, + } + } + + pub(super) const fn deleted_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_DELETED, + Self::Workflow => &WORKFLOW_DELETED, + } + } + + pub(super) const fn present_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown present paths", + Self::Workflow => "git workflow present paths", + } + } + + pub(super) const fn deleted_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown deleted paths", + Self::Workflow => "git workflow deleted paths", + } + } +} diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs new file mode 100644 index 0000000..40e39f3 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -0,0 +1,141 @@ +//! This module owns retained identity evidence for one documentation source. + +use std::fs::File; +use std::fs::{self, OpenOptions}; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use std::path::PathBuf; + +use super::CorpusKind; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::{ + OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot, copy_exact, +}; + +pub(super) struct AdmittedSource { + file: File, + identity: RepositoryFileIdentity, + path: String, + relative: PathBuf, +} + +impl AdmittedSource { + pub(super) fn admit( + file: File, + path: String, + relative: PathBuf, + kind: CorpusKind, + ) -> Result { + let identity = identity(&file, kind, &path)?; + Ok(Self { + file, + identity, + path, + relative, + }) + } + + pub(super) const fn bytes(&self) -> u64 { + self.identity.bytes() + } + + pub(super) fn path(&self) -> &str { + &self.path + } + + /// Streams the exact admitted bytes into a new read-only snapshot file. + /// + /// The retained descriptor identity is checked before and after copying. + /// The copy uses a fixed 16 KiB buffer and checked offset arithmetic; it + /// performs no durability synchronization because the snapshot is consumed + /// only by the current validation process. + pub(super) fn materialize( + &self, + snapshot_root: &Path, + kind: CorpusKind, + ) -> Result<(), DocumentationError> { + self.verify_retained(kind)?; + let destination = snapshot_root.join(&self.relative); + let parent = destination.parent().ok_or_else(|| { + snapshot_io( + "resolve documentation snapshot parent", + io::Error::other("source path has no parent"), + ) + })?; + fs::create_dir_all(parent) + .map_err(|source| snapshot_io("create documentation snapshot directory", source))?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(&destination) + .map_err(|source| snapshot_io("create documentation snapshot source", source))?; + copy_exact(&self.file, &mut output, self.identity.bytes()) + .map_err(|source| snapshot_io("copy documentation snapshot source", source))?; + self.verify_retained(kind) + } + + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + kind: CorpusKind, + ) -> Result<(), DocumentationError> { + let current = match repository_root.open_file(&self.relative) { + Ok(file) => file, + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(changed(kind, &self.path)); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(DocumentationError::Inspect { + corpus: kind.label(), + path: self.path.clone(), + source, + }); + } + Err(OpenRepositoryFileError::NonRegular) => { + return Err(changed(kind, &self.path)); + } + }; + let current = identity(¤t, kind, &self.path)?; + if current == self.identity { + Ok(()) + } else { + Err(changed(kind, &self.path)) + } + } + + fn verify_retained(&self, kind: CorpusKind) -> Result<(), DocumentationError> { + let current = identity(&self.file, kind, &self.path)?; + if current == self.identity { + Ok(()) + } else { + Err(changed(kind, &self.path)) + } + } +} + +fn identity( + file: &File, + kind: CorpusKind, + path: &str, +) -> Result { + RepositoryFileIdentity::read(file).map_err(|source| DocumentationError::Inspect { + corpus: kind.label(), + path: path.to_owned(), + source, + }) +} + +fn changed(kind: CorpusKind, path: &str) -> DocumentationError { + DocumentationError::CorpusChanged { + corpus: kind.label(), + path: path.to_owned(), + } +} + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs new file mode 100644 index 0000000..82e43c2 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -0,0 +1,273 @@ +//! This module owns documentation source-corpus regression evidence. + +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; +use std::process::Command; + +use super::{CorpusKind, SourceCorpus, admit_path}; +use crate::documentation_integrity::error::DocumentationError; +use crate::git_inventory::GitPath; +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +#[test] +fn markdown_corpus_is_the_sorted_present_repository_set() -> Result<(), Box> +{ + let directory = TestDirectory::create("documentation-markdown-corpus")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + write(root, ".gitignore", "/target/\n")?; + write(root, "zulu.md", "# Zulu\n")?; + write(root, "alpha.md", "# Alpha\n")?; + write(root, "deleted.md", "# Deleted\n")?; + write(root, "target/generated.md", "# Generated\n")?; + run_git(root, &["add", ".gitignore", "zulu.md", "deleted.md"])?; + fs::remove_file(root.join("deleted.md"))?; + + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + assert_eq!(corpus.paths(), ["alpha.md", "zulu.md"]); + directory.close()?; + Ok(()) +} + +#[test] +fn workflow_corpus_is_the_sorted_present_repository_set() -> Result<(), Box> +{ + let directory = TestDirectory::create("documentation-workflow-corpus")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + write(root, ".gitignore", "/.github/workflows/generated.yml\n")?; + write(root, ".github/workflows/zulu.yml", "name: Zulu\n")?; + write(root, ".github/workflows/alpha.yaml", "name: Alpha\n")?; + write(root, ".github/workflows/deleted.yml", "name: Deleted\n")?; + write(root, ".github/workflows/generated.yml", "name: Generated\n")?; + run_git( + root, + &[ + "add", + ".gitignore", + ".github/workflows/zulu.yml", + ".github/workflows/deleted.yml", + ], + )?; + fs::remove_file(root.join(".github/workflows/deleted.yml"))?; + + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::workflow(&repository_root, &process_directory)?; + + assert_eq!( + corpus.paths(), + [".github/workflows/alpha.yaml", ".github/workflows/zulu.yml"] + ); + directory.close()?; + Ok(()) +} + +#[test] +fn repository_configured_global_ignores_cannot_change_the_corpus() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-global-ignore")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + write(root, "tracked.md", "# Tracked\n")?; + write(root, "new.md", "# New\n")?; + write(root, "global-ignore", "*.md\n")?; + run_git(root, &["add", "tracked.md"])?; + let global_ignore = root.join("global-ignore"); + let global_ignore = global_ignore + .to_str() + .ok_or("test path is not valid Unicode")?; + run_git(root, &["config", "core.excludesFile", global_ignore])?; + + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + assert_eq!(corpus.paths(), ["new.md", "tracked.md"]); + directory.close()?; + Ok(()) +} + +#[test] +fn transient_repository_replacement_cannot_redirect_the_corpus() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-root-replacement")?; + let root = directory.path().join("repository"); + let retained = directory.path().join("retained"); + fs::create_dir(&root)?; + run_git(&root, &["init", "--quiet", "--template="])?; + write(&root, "original.md", "# Original\n")?; + let repository_root = RepositoryRoot::open(&root)?; + let process_directory = repository_root.process_directory()?; + let original_process_directory = std::env::current_dir()?; + + fs::rename(&root, &retained)?; + fs::create_dir(&root)?; + run_git(&root, &["init", "--quiet", "--template="])?; + write(&root, "substitute.md", "# Substitute\n")?; + + let corpus = SourceCorpus::markdown(&repository_root, &process_directory); + fs::remove_dir_all(&root)?; + fs::rename(&retained, &root)?; + let corpus = corpus?; + + assert_eq!(corpus.paths(), ["original.md"]); + assert_eq!(std::env::current_dir()?, original_process_directory); + drop(repository_root); + directory.close()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn symlinked_markdown_is_refused() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let directory = TestDirectory::create("documentation-markdown-symlink")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + write(root, "target.txt", "target\n")?; + symlink("target.txt", root.join("linked.md"))?; + + let repository_root = RepositoryRoot::open(root)?; + assert!(matches!( + repository_root.open_file(Path::new("linked.md")), + Err(OpenRepositoryFileError::NonRegular) + )); + let process_directory = repository_root.process_directory()?; + let result = SourceCorpus::markdown(&repository_root, &process_directory); + + assert!(matches!( + result, + Err(DocumentationError::NonRegular { + corpus: "Markdown", + ref path, + }) if path == "linked.md" + )); + directory.close()?; + Ok(()) +} + +#[test] +fn oversized_markdown_is_refused_before_external_validation() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-markdown-size")?; + let root = directory.path(); + let path = root.join("oversized.md"); + let observed = 4_194_304_u64 + .checked_add(1) + .ok_or("corpus file bound overflow")?; + fs::File::create(&path)?.set_len(observed)?; + let repository_root = RepositoryRoot::open(root)?; + let git_path = GitPath::new(b"oversized.md".to_vec()); + + let paths = BTreeSet::from([git_path]); + let result = super::admit_paths(&repository_root, paths.iter(), CorpusKind::Markdown); + + assert!(matches!( + result, + Err(DocumentationError::CorpusFileTooLarge { + corpus: "Markdown", + ref path, + maximum: super::CORPUS_FILE_MAX_BYTES, + observed: actual, + }) if path == "oversized.md" && actual == observed + )); + directory.close()?; + Ok(()) +} + +#[test] +fn aggregate_documentation_bytes_are_bounded_with_checked_accounting() +-> Result<(), Box> { + let mut budget = super::CorpusByteBudget::default(); + for _ in 0..16 { + budget.admit( + CorpusKind::Markdown, + "bounded.md", + super::CORPUS_FILE_MAX_BYTES, + )?; + } + let observed = super::CORPUS_MAX_BYTES + .checked_add(super::CORPUS_FILE_MAX_BYTES) + .ok_or("aggregate corpus bound overflow")?; + + assert!(matches!( + budget.admit( + CorpusKind::Markdown, + "bounded.md", + super::CORPUS_FILE_MAX_BYTES, + ), + Err(DocumentationError::CorpusTooLarge { + corpus: "Markdown", + maximum: super::CORPUS_MAX_BYTES, + observed: actual, + }) if actual == observed + )); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn fifo_workflow_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-workflow-fifo")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + write(root, ".github/workflows/blocking.yml", "name: Blocking\n")?; + run_git(root, &["add", ".github/workflows/blocking.yml"])?; + let fifo = root.join(".github/workflows/blocking.yml"); + fs::remove_file(&fifo)?; + let status = Command::new("mkfifo").arg(&fifo).status()?; + if !status.success() { + return Err("mkfifo fixture command failed".into()); + } + + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let result = SourceCorpus::workflow(&repository_root, &process_directory); + + assert!(matches!( + result, + Err(DocumentationError::NonRegular { + corpus: "GitHub Actions workflow", + ref path, + }) if path == ".github/workflows/blocking.yml" + )); + directory.close()?; + Ok(()) +} + +#[test] +fn non_utf8_markdown_path_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-markdown-non-utf8")?; + let root = directory.path(); + let path = GitPath::new(b"bad\xff.md".to_vec()); + + let repository_root = RepositoryRoot::open(root)?; + let result = admit_path(&repository_root, &path, CorpusKind::Markdown); + + assert!(matches!( + result, + Err(DocumentationError::PathEncoding { + corpus: "Markdown", + .. + }) + )); + directory.close()?; + Ok(()) +} + +fn write(root: &Path, relative: &str, contents: &str) -> Result<(), Box> { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, contents)?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs new file mode 100644 index 0000000..1f6f8e8 --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -0,0 +1,164 @@ +//! This module owns Dependabot manifest coverage and maintenance policy. + +mod manifest; + +use std::collections::BTreeSet; + +use yaml_rust2::{Yaml, YamlLoader}; + +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +use super::error::DocumentationError; +use super::repository_text::{self, RepositoryText}; +use manifest::tracked_scopes; + +const DEPENDABOT_PATH: &str = ".github/dependabot.yml"; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct DependencyScope { + ecosystem: String, + directory: String, +} + +pub(super) fn check( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, +) -> Result { + let raw = repository_text::read(repository_root, DEPENDABOT_PATH)?; + let required = tracked_scopes(process_directory)?; + admit(raw.as_str(), &required)?; + raw.verify(repository_root)?; + Ok(raw) +} + +fn admit(raw: &str, required: &BTreeSet) -> Result<(), DocumentationError> { + let documents = + YamlLoader::load_from_str(raw).map_err(|source| DocumentationError::RepositoryYaml { + path: DEPENDABOT_PATH, + source, + })?; + let [document] = documents.as_slice() else { + return Err(contract("policy contains exactly one YAML document")); + }; + if document["version"].as_i64() != Some(2) { + return Err(contract("policy version is exactly 2")); + } + let Some(updates) = document["updates"].as_vec() else { + return Err(contract("updates is a sequence")); + }; + if updates.is_empty() { + return Err(contract("at least one update block exists")); + } + let mut configured = BTreeSet::new(); + for update in updates { + let scopes = block_scopes(update)?; + admit_maintenance_policy(update, &scopes)?; + for scope in scopes { + if !configured.insert(scope.clone()) { + return Err(contract_at( + scope.diagnostic(), + "update scope appears exactly once", + )); + } + } + } + if let Some(missing) = required.difference(&configured).next() { + return Err(contract_at( + missing.diagnostic(), + "tracked dependency scope has an update policy", + )); + } + Ok(()) +} + +fn block_scopes(update: &Yaml) -> Result, DocumentationError> { + let ecosystem = update["package-ecosystem"] + .as_str() + .ok_or_else(|| contract("every update block names an ecosystem"))?; + let has_directory = !update["directory"].is_badvalue(); + let has_directories = !update["directories"].is_badvalue(); + if has_directory && has_directories { + return Err(contract("update block chooses one directory form")); + } + let mut scopes = Vec::new(); + if has_directory { + let directory = update["directory"] + .as_str() + .ok_or_else(|| contract("update directory is a string"))?; + scopes.push(DependencyScope::new(ecosystem, directory)); + } + if has_directories { + let directories = update["directories"] + .as_vec() + .ok_or_else(|| contract("update directories is a sequence"))?; + for directory in directories { + let directory = directory + .as_str() + .ok_or_else(|| contract("every update directory is a string"))?; + scopes.push(DependencyScope::new(ecosystem, directory)); + } + } + if scopes.is_empty() { + Err(contract_at( + ecosystem.to_owned(), + "update block names at least one directory", + )) + } else { + Ok(scopes) + } +} + +fn admit_maintenance_policy( + update: &Yaml, + scopes: &[DependencyScope], +) -> Result<(), DocumentationError> { + let labels_are_exact = update["labels"].as_vec().is_some_and( + |labels| matches!(labels.as_slice(), [label] if label.as_str() == Some("dependencies")), + ); + let uniform = update["schedule"]["interval"].as_str() == Some("weekly") + && update["open-pull-requests-limit"].as_i64() == Some(5) + && labels_are_exact; + if uniform { + Ok(()) + } else { + let subject = scopes + .first() + .map_or_else(String::new, DependencyScope::diagnostic); + Err(contract_at( + subject, + "update block uses the maintenance policy", + )) + } +} + +impl DependencyScope { + fn new(ecosystem: &str, directory: &str) -> Self { + Self { + ecosystem: ecosystem.to_owned(), + directory: directory.to_owned(), + } + } + + fn diagnostic(&self) -> String { + format!("{} {}", self.ecosystem, self.directory) + } +} + +const fn contract(requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { + path: DEPENDABOT_PATH, + requirement, + } +} + +const fn contract_at(subject: String, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContractAt { + path: DEPENDABOT_PATH, + subject, + requirement, + } +} + +#[cfg(test)] +#[path = "dependabot/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/dependabot/manifest.rs b/xtask/src/documentation_integrity/dependabot/manifest.rs new file mode 100644 index 0000000..defb753 --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot/manifest.rs @@ -0,0 +1,62 @@ +//! This module owns tracked dependency-manifest scope discovery. + +use std::collections::BTreeSet; +use xtask::protocol_admission::posix_relative_path; + +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::RepositoryProcessDirectory; + +use super::DependencyScope; +use crate::documentation_integrity::error::DocumentationError; + +const MANIFEST_ARGUMENTS: [&str; 5] = [ + "ls-files", + "-z", + "--", + ":(glob)**/Cargo.toml", + ":(glob)**/package.json", +]; + +pub(super) fn tracked_scopes( + process_directory: &RepositoryProcessDirectory, +) -> Result, DocumentationError> { + paths_with( + &MANIFEST_ARGUMENTS, + "list tracked dependency manifests", + |command| process_directory.spawn(command), + )? + .iter() + .map(manifest_scope) + .chain([Ok(DependencyScope::new("github-actions", "/"))]) + .collect() +} + +fn manifest_scope(path: &GitPath) -> Result { + let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { + DocumentationError::PathEncoding { + corpus: "dependency manifest", + source, + } + })?; + posix_relative_path(&text).map_err(|_| DocumentationError::InvalidPath { + corpus: "dependency manifest", + path: text.clone(), + })?; + let (directory, filename) = text.rsplit_once('/').unwrap_or(("", &text)); + let ecosystem = match filename { + "Cargo.toml" => "cargo", + "package.json" => "npm", + _ => { + return Err(DocumentationError::InvalidPath { + corpus: "dependency manifest", + path: text, + }); + } + }; + let directory = if directory.is_empty() { + String::from("/") + } else { + format!("/{directory}") + }; + Ok(DependencyScope::new(ecosystem, &directory)) +} diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs new file mode 100644 index 0000000..28b6334 --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -0,0 +1,152 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +use super::DependencyScope; + +const POLICY: &str = r"version: 2 +updates: + - package-ecosystem: cargo + directories: + - / + - /xtask + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: + - dependencies + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: + - dependencies +"; + +fn required() -> BTreeSet { + [ + DependencyScope::new("cargo", "/"), + DependencyScope::new("cargo", "/xtask"), + DependencyScope::new("github-actions", "/"), + ] + .into_iter() + .collect() +} + +#[test] +fn complete_uniform_dependabot_policy_is_admitted() { + assert!(super::admit(POLICY, &required()).is_ok()); +} + +#[test] +fn update_block_with_both_directory_forms_is_refused() { + let policy = POLICY.replacen(" - /xtask\n", "", 1).replacen( + " schedule:\n", + " directory: /xtask\n schedule:\n", + 1, + ); + + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContract { + path: super::DEPENDABOT_PATH, + requirement: "update block chooses one directory form", + }) + )); +} + +#[test] +fn missing_manifest_scope_is_refused() { + let policy = POLICY.replace(" - /xtask\n", ""); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "tracked dependency scope has an update policy", + }) if subject == "cargo /xtask" + )); +} + +#[test] +fn duplicate_update_scope_is_refused() { + let policy = POLICY.replace(" - /xtask\n", " - /xtask\n - /xtask\n"); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update scope appears exactly once", + }) if subject == "cargo /xtask" + )); +} + +#[test] +fn duplicate_yaml_mapping_keys_are_refused_before_policy_admission() { + let top_level = format!("{POLICY}updates: []\n"); + let nested = POLICY.replacen( + " interval: weekly\n", + " interval: weekly\n interval: daily\n", + 1, + ); + for policy in [top_level, nested] { + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryYaml { + path: super::DEPENDABOT_PATH, + .. + }) + )); + } +} + +#[test] +fn nonuniform_maintenance_policy_is_refused() { + let policy = POLICY.replacen(" interval: weekly", " interval: daily", 1); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update block uses the maintenance policy", + }) if subject == "cargo /" + )); +} + +#[test] +fn prefix_confusable_maintenance_values_are_refused() { + for policy in [ + POLICY.replacen(" interval: weekly", " interval: weeklylies", 1), + POLICY.replacen( + " open-pull-requests-limit: 5", + " open-pull-requests-limit: 50", + 1, + ), + POLICY.replacen(" - dependencies", " - dependencies-extra", 1), + ] { + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update block uses the maintenance policy", + }) if subject == "cargo /" + )); + } +} + +#[test] +fn committed_dependabot_policy_covers_every_tracked_manifest() +-> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + super::check(&repository_root, &process_directory)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs new file mode 100644 index 0000000..25410ed --- /dev/null +++ b/xtask/src/documentation_integrity/error.rs @@ -0,0 +1,207 @@ +//! This module owns typed documentation-integrity failures. + +mod display; + +use std::error::Error; +use std::fmt; +use std::io; +use std::path::PathBuf; +use std::string::FromUtf8Error; + +use crate::bounded_process::ProcessError; +use crate::git_inventory::GitInventoryError; + +pub(crate) enum DocumentationError { + CheckFailures { + first: Box, + second: Box, + }, + /// One selected source's metadata length exceeds the per-file byte bound. + CorpusFileTooLarge { + /// Human-readable corpus kind whose source was refused. + corpus: &'static str, + /// Repository-relative source path reported by Git. + path: String, + /// Largest admitted metadata length in bytes. + maximum: u64, + /// Observed metadata length in bytes. + observed: u64, + }, + /// Checked aggregate byte accounting overflowed for the named corpus kind. + CorpusSizeOverflow(&'static str), + /// The selected corpus's checked aggregate length exceeds its byte bound. + CorpusTooLarge { + /// Human-readable corpus kind whose aggregate was refused. + corpus: &'static str, + /// Largest admitted aggregate length in bytes. + maximum: u64, + /// Observed checked aggregate length in bytes. + observed: u64, + }, + /// A retained source identity or the exact selected path set changed. + CorpusChanged { + /// Human-readable corpus kind whose invariant changed. + corpus: &'static str, + /// Repository-relative source path identifying the first difference. + path: String, + }, + /// Git selected no present sources for the named required corpus kind. + EmptyCorpus(&'static str), + EnvironmentUnavailable(&'static str), + GitInventory(GitInventoryError), + Inspect { + corpus: &'static str, + path: String, + source: io::Error, + }, + InvalidPath { + corpus: &'static str, + path: String, + }, + NonRegular { + corpus: &'static str, + path: String, + }, + PathEncoding { + corpus: &'static str, + source: FromUtf8Error, + }, + Process(ProcessError), + /// A filesystem operation could not construct or remove a refusal fixture. + RefusalFixture { + action: &'static str, + source: io::Error, + }, + /// A malformed-input scenario did not produce its exact reviewed refusal. + RefusalMismatch { + scenario: &'static str, + observed: Option>, + }, + RepositoryFileEncoding { + path: &'static str, + source: FromUtf8Error, + }, + RepositoryFileInspect { + path: &'static str, + source: io::Error, + }, + /// A fixed policy path no longer names the file identity admitted for validation. + RepositoryFileChanged(&'static str), + RepositoryFileNonRegular(&'static str), + RepositoryFileTooLarge { + path: &'static str, + maximum: u64, + }, + RepositoryContract { + path: &'static str, + requirement: &'static str, + }, + RepositoryContractAt { + path: &'static str, + subject: String, + requirement: &'static str, + }, + RepositoryJson { + path: &'static str, + source: serde_json::Error, + }, + RepositoryYaml { + path: &'static str, + source: yaml_rust2::ScanError, + }, + RepositoryRootChanged(PathBuf), + RepositoryRootInspect { + path: PathBuf, + source: io::Error, + }, + RepositoryValue { + path: &'static str, + field: &'static str, + expected: &'static str, + observed: Option, + }, + /// A filesystem operation could not construct or remove a tool-input snapshot. + Snapshot { + /// Snapshot operation that could not complete. + action: &'static str, + /// Underlying filesystem failure. + source: io::Error, + }, + VersionMismatch { + program: &'static str, + expected: &'static str, + observed: String, + }, + ToolFailed { + program: &'static str, + code: Option, + stdout: String, + stderr: String, + }, + ToolOutputEncoding { + program: &'static str, + stream: &'static str, + source: FromUtf8Error, + }, + ToolUnavailable { + program: &'static str, + install_version: &'static str, + source: ProcessError, + }, +} + +impl fmt::Debug for DocumentationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl Error for DocumentationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::CheckFailures { first, .. } => Some(first), + Self::GitInventory(error) => Some(error), + Self::Process(error) => Some(error), + Self::Inspect { source, .. } + | Self::RefusalFixture { source, .. } + | Self::RepositoryFileInspect { source, .. } + | Self::RepositoryRootInspect { source, .. } + | Self::Snapshot { source, .. } => Some(source), + Self::PathEncoding { source, .. } | Self::RepositoryFileEncoding { source, .. } => { + Some(source) + } + Self::RefusalMismatch { + observed: Some(error), + .. + } => Some(error), + Self::RepositoryJson { source, .. } => Some(source), + Self::RepositoryYaml { source, .. } => Some(source), + Self::ToolOutputEncoding { source, .. } => Some(source), + Self::ToolUnavailable { source, .. } => Some(source), + Self::CorpusFileTooLarge { .. } + | Self::CorpusSizeOverflow(_) + | Self::CorpusTooLarge { .. } + | Self::CorpusChanged { .. } + | Self::EmptyCorpus(_) + | Self::EnvironmentUnavailable(_) + | Self::InvalidPath { .. } + | Self::NonRegular { .. } + | Self::RefusalMismatch { observed: None, .. } + | Self::RepositoryFileChanged(_) + | Self::RepositoryFileNonRegular(_) + | Self::RepositoryFileTooLarge { .. } + | Self::RepositoryContract { .. } + | Self::RepositoryContractAt { .. } + | Self::RepositoryRootChanged(_) + | Self::RepositoryValue { .. } + | Self::ToolFailed { .. } + | Self::VersionMismatch { .. } => None, + } + } +} + +impl From for DocumentationError { + fn from(error: GitInventoryError) -> Self { + Self::GitInventory(error) + } +} diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs new file mode 100644 index 0000000..5e675bd --- /dev/null +++ b/xtask/src/documentation_integrity/error/display.rs @@ -0,0 +1,293 @@ +//! This module owns human-readable documentation-integrity diagnostics. + +use std::fmt; + +use crate::diagnostic::{escaped_controls, escaped_path}; + +use super::DocumentationError; + +mod refusal; + +use refusal::{refusal_fixture, refusal_mismatch}; + +#[derive(Clone, Copy)] +enum SourcePathDiagnostic { + Changed, + Inspect, + Invalid, + NonRegular, +} + +#[derive(Clone, Copy)] +enum RepositoryRootDiagnostic { + Changed, + Inspect, +} + +impl fmt::Display for DocumentationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CheckFailures { first, second } => { + write!(formatter, "{first}; additionally: {second}") + } + error @ (Self::CorpusFileTooLarge { .. } + | Self::CorpusSizeOverflow(_) + | Self::CorpusTooLarge { .. } + | Self::CorpusChanged { .. } + | Self::EmptyCorpus(_)) => corpus(formatter, error), + Self::EnvironmentUnavailable(variable) => { + write!( + formatter, + "documentation tool environment variable `{variable}` is unavailable" + ) + } + Self::GitInventory(error) => write!(formatter, "{error}"), + Self::Inspect { corpus, path, .. } => { + source_path(formatter, SourcePathDiagnostic::Inspect, corpus, path) + } + Self::InvalidPath { corpus, path } => { + source_path(formatter, SourcePathDiagnostic::Invalid, corpus, path) + } + Self::NonRegular { corpus, path } => { + source_path(formatter, SourcePathDiagnostic::NonRegular, corpus, path) + } + Self::PathEncoding { corpus, .. } => { + write!(formatter, "{corpus} corpus contains a non-UTF-8 path") + } + Self::Process(error) => write!(formatter, "{error}"), + Self::RefusalFixture { action, .. } => refusal_fixture(formatter, action), + Self::RefusalMismatch { scenario, observed } => { + refusal_mismatch(formatter, scenario, observed.as_deref()) + } + error @ (Self::RepositoryFileEncoding { .. } + | Self::RepositoryFileChanged(_) + | Self::RepositoryFileInspect { .. } + | Self::RepositoryFileNonRegular(_) + | Self::RepositoryFileTooLarge { .. } + | Self::RepositoryContract { .. } + | Self::RepositoryContractAt { .. } + | Self::RepositoryJson { .. } + | Self::RepositoryYaml { .. } + | Self::RepositoryValue { .. }) => repository_file(formatter, error), + Self::RepositoryRootChanged(path) => { + repository_root(formatter, RepositoryRootDiagnostic::Changed, path) + } + Self::RepositoryRootInspect { path, .. } => { + repository_root(formatter, RepositoryRootDiagnostic::Inspect, path) + } + Self::Snapshot { action, .. } => { + write!(formatter, "cannot {action}") + } + error @ (Self::VersionMismatch { .. } + | Self::ToolFailed { .. } + | Self::ToolOutputEncoding { .. } + | Self::ToolUnavailable { .. }) => tool(formatter, error), + } + } +} + +fn corpus(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::CorpusFileTooLarge { + corpus, + path, + maximum, + observed, + } => write!( + formatter, + "{corpus} source `{path}` is {observed} bytes; maximum is {maximum}" + ), + DocumentationError::CorpusSizeOverflow(corpus) => { + write!(formatter, "{corpus} corpus byte count overflowed") + } + DocumentationError::CorpusTooLarge { + corpus, + maximum, + observed, + } => write!( + formatter, + "{corpus} corpus is {observed} bytes; maximum is {maximum}" + ), + DocumentationError::CorpusChanged { corpus, path } => { + source_path(formatter, SourcePathDiagnostic::Changed, corpus, path) + } + DocumentationError::EmptyCorpus(label) => { + write!(formatter, "the {label} corpus is empty") + } + _ => Err(fmt::Error), + } +} + +fn repository_file(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::RepositoryFileChanged(path) => { + write!( + formatter, + "repository file changed during validation: `{path}`" + ) + } + DocumentationError::RepositoryFileEncoding { path, .. } => { + write!(formatter, "repository file `{path}` is not UTF-8") + } + DocumentationError::RepositoryFileInspect { path, .. } => { + write!(formatter, "cannot inspect repository file `{path}`") + } + DocumentationError::RepositoryFileNonRegular(path) => { + write!(formatter, "repository file is not regular: `{path}`") + } + DocumentationError::RepositoryFileTooLarge { path, maximum } => write!( + formatter, + "repository file `{path}` exceeds the {maximum}-byte bound" + ), + DocumentationError::RepositoryContract { path, requirement } => { + write!( + formatter, + "repository file `{path}` violates: {requirement}" + ) + } + DocumentationError::RepositoryContractAt { + path, + subject, + requirement, + } => repository_contract_at(formatter, path, subject, requirement), + DocumentationError::RepositoryJson { path, .. } => { + write!(formatter, "repository file `{path}` is not valid JSON") + } + DocumentationError::RepositoryYaml { path, .. } => { + write!(formatter, "repository file `{path}` is not valid YAML") + } + DocumentationError::RepositoryValue { + path, + field, + expected, + observed, + } => repository_value(formatter, path, field, expected, observed.as_deref()), + _ => Err(fmt::Error), + } +} + +fn tool(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::VersionMismatch { + program, + expected, + observed, + } => write!( + formatter, + "{program} version mismatch: expected {expected:?}, observed {observed:?}" + ), + DocumentationError::ToolFailed { + program, + code, + stdout, + stderr, + } => tool_failed(formatter, program, *code, stdout, stderr), + DocumentationError::ToolOutputEncoding { + program, stream, .. + } => { + write!(formatter, "{program} {stream} is not UTF-8") + } + DocumentationError::ToolUnavailable { + program, + install_version, + .. + } => write!( + formatter, + "{program} is unavailable; install version {install_version}" + ), + _ => Err(fmt::Error), + } +} + +fn source_path( + formatter: &mut fmt::Formatter<'_>, + diagnostic: SourcePathDiagnostic, + corpus: &str, + path: &str, +) -> fmt::Result { + match diagnostic { + SourcePathDiagnostic::Changed => { + write!(formatter, "{corpus} source changed during validation: `")?; + } + SourcePathDiagnostic::Inspect => write!(formatter, "cannot inspect {corpus} source `")?, + SourcePathDiagnostic::Invalid => { + write!(formatter, "{corpus} corpus contains an unsafe path `")?; + } + SourcePathDiagnostic::NonRegular => { + write!(formatter, "{corpus} source is not a regular file: `")?; + } + } + escaped_controls(formatter, path)?; + formatter.write_str("`") +} + +fn repository_contract_at( + formatter: &mut fmt::Formatter<'_>, + path: &str, + subject: &str, + requirement: &str, +) -> fmt::Result { + write!( + formatter, + "repository file `{path}` violates {requirement} at `" + )?; + escaped_controls(formatter, subject)?; + formatter.write_str("`") +} + +fn repository_root( + formatter: &mut fmt::Formatter<'_>, + diagnostic: RepositoryRootDiagnostic, + path: &std::path::Path, +) -> fmt::Result { + match diagnostic { + RepositoryRootDiagnostic::Changed => { + formatter.write_str("documentation repository root changed: `")?; + } + RepositoryRootDiagnostic::Inspect => { + formatter.write_str("cannot inspect documentation repository root: `")?; + } + } + escaped_path(formatter, path)?; + formatter.write_str("`") +} + +fn repository_value( + formatter: &mut fmt::Formatter<'_>, + path: &str, + field: &str, + expected: &str, + observed: Option<&str>, +) -> fmt::Result { + write!( + formatter, + "repository file `{path}` requires `{field}` to be {expected:?}; observed " + )?; + match observed { + Some(value) => { + formatter.write_str("\"")?; + escaped_controls(formatter, value)?; + formatter.write_str("\"") + } + None => formatter.write_str("missing"), + } +} + +fn tool_failed( + formatter: &mut fmt::Formatter<'_>, + program: &str, + code: Option, + stdout: &str, + stderr: &str, +) -> fmt::Result { + write!(formatter, "{program} failed with exit status ")?; + match code { + Some(code) => write!(formatter, "{code}")?, + None => formatter.write_str("unavailable")?, + } + formatter.write_str("; stdout \"")?; + escaped_controls(formatter, stdout)?; + formatter.write_str("\"; stderr \"")?; + escaped_controls(formatter, stderr)?; + formatter.write_str("\"") +} diff --git a/xtask/src/documentation_integrity/error/display/refusal.rs b/xtask/src/documentation_integrity/error/display/refusal.rs new file mode 100644 index 0000000..1ee584e --- /dev/null +++ b/xtask/src/documentation_integrity/error/display/refusal.rs @@ -0,0 +1,37 @@ +//! This module owns human-readable documentation-refusal diagnostics. + +use std::fmt; + +use super::super::DocumentationError; + +/// Formats one failed filesystem action at the refusal-fixture boundary. +/// +/// `action` names the attempted operation without terminal punctuation. The +/// formatter receives no underlying I/O details; those remain available through +/// the error source chain. +pub(super) fn refusal_fixture(formatter: &mut fmt::Formatter<'_>, action: &str) -> fmt::Result { + write!( + formatter, + "cannot {action} for documentation refusal evidence" + ) +} + +/// Formats a refusal scenario that did not return its exact reviewed error. +/// +/// The stable prefix identifies `scenario`. When `observed` contains a different +/// typed failure, its escaped display diagnostic is appended as secondary +/// evidence; normal absence records that the malformed input was accepted. +pub(super) fn refusal_mismatch( + formatter: &mut fmt::Formatter<'_>, + scenario: &str, + observed: Option<&DocumentationError>, +) -> fmt::Result { + write!( + formatter, + "documentation refusal scenario `{scenario}` did not produce its reviewed error" + )?; + if let Some(error) = observed { + write!(formatter, ": {error}")?; + } + Ok(()) +} diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs new file mode 100644 index 0000000..b2aff9a --- /dev/null +++ b/xtask/src/documentation_integrity/execution.rs @@ -0,0 +1,237 @@ +//! This module owns bounded execution of admitted documentation tools. + +mod corpus_guard; +mod refusal_check; +mod snapshot; + +use std::env; +use std::ffi::OsStr; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::bounded_process::{self, ProcessOutput}; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +use super::corpus::SourceCorpus; +use super::error::DocumentationError; +use super::tool::DocumentationTool; +use corpus_guard::CorpusGuardedRunner; + +const TOOL_DEADLINE: Duration = Duration::from_mins(2); + +trait ToolRunner { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result; +} + +trait DirectoryToolRunner { + fn capture_in( + &mut self, + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], + ) -> Result; +} + +struct ExternalToolRunner<'a> { + process_directory: &'a RepositoryProcessDirectory, +} + +pub(super) fn run( + process_directory: &RepositoryProcessDirectory, + repository_root: &RepositoryRoot, + markdown: &SourceCorpus, + workflows: &SourceCorpus, +) -> Result<(), DocumentationError> { + let corpora = [markdown, workflows]; + let external = ExternalToolRunner { process_directory }; + let mut runner = + CorpusGuardedRunner::new(external, process_directory, repository_root, &corpora)?; + let result = run_with(&mut runner, markdown.paths(), workflows.paths()); + let cleanup = runner.close(); + finish_run(result, cleanup) +} + +fn finish_run( + result: Result<(), DocumentationError>, + cleanup: Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { + combine_checks(result, cleanup) +} + +/// Executes both named malformed-input scenarios through the production runner. +pub(super) fn check_refusals() -> Result<(), DocumentationError> { + refusal_check::check() +} + +fn run_with( + runner: &mut impl ToolRunner, + markdown: &[String], + workflows: &[String], +) -> Result<(), DocumentationError> { + admit_version(runner, DocumentationTool::Markdownlint)?; + admit_version(runner, DocumentationTool::Lychee)?; + let lint = run_check(runner, DocumentationTool::Markdownlint, markdown); + let links = run_check(runner, DocumentationTool::Lychee, markdown); + combine_checks(lint, links)?; + admit_version(runner, DocumentationTool::Actionlint)?; + run_check(runner, DocumentationTool::Actionlint, workflows) +} + +fn combine_checks( + first: Result<(), DocumentationError>, + second: Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { + match (first, second) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(first), Err(second)) => Err(DocumentationError::CheckFailures { + first: Box::new(first), + second: Box::new(second), + }), + } +} + +fn admit_version( + runner: &mut impl ToolRunner, + tool: DocumentationTool, +) -> Result<(), DocumentationError> { + let arguments = text_arguments(tool.version_arguments()); + let output = runner.capture(tool, &arguments)?; + let stdout = String::from_utf8(output.stdout).map_err(|source| { + DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "version stdout", + source, + } + })?; + let observed = stdout.lines().next().unwrap_or(""); + let admission = tool.admit_version(observed); + if output.succeeded { + admission + } else { + admission?; + Err(failed(tool, output.code, stdout, output.stderr)?) + } +} + +fn run_check( + runner: &mut impl ToolRunner, + tool: DocumentationTool, + paths: &[String], +) -> Result<(), DocumentationError> { + let mut arguments = text_arguments(tool.check_prefix()); + arguments.extend(paths.iter().cloned()); + let output = runner.capture(tool, &arguments)?; + if output.succeeded { + Ok(()) + } else { + Err(failed( + tool, + output.code, + String::from_utf8(output.stdout).map_err(|source| { + DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "stdout", + source, + } + })?, + output.stderr, + )?) + } +} + +fn failed( + tool: DocumentationTool, + code: Option, + stdout: String, + stderr: Vec, +) -> Result { + let stderr = + String::from_utf8(stderr).map_err(|source| DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "stderr", + source, + })?; + Ok(DocumentationError::ToolFailed { + program: tool.program(), + code, + stdout, + stderr, + }) +} + +fn text_arguments(arguments: &[&str]) -> Vec { + arguments.iter().map(ToString::to_string).collect() +} + +impl ToolRunner for ExternalToolRunner<'_> { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + capture_external(self.process_directory, tool, arguments) + } +} + +impl DirectoryToolRunner for ExternalToolRunner<'_> { + fn capture_in( + &mut self, + _repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + capture_external(process_directory, tool, arguments) + } +} + +fn capture_external( + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], +) -> Result { + let path = env::var_os("PATH").ok_or(DocumentationError::EnvironmentUnavailable("PATH"))?; + let mut command = documentation_command(tool, arguments, &path); + bounded_process::capture_with( + tool.program(), + &mut command, + Some(TOOL_DEADLINE), + |command| process_directory.spawn(command), + ) + .map_err(|source| { + if source.is_not_found() { + DocumentationError::ToolUnavailable { + program: tool.program(), + install_version: tool.install_version(), + source, + } + } else { + DocumentationError::Process(source) + } + }) +} + +fn documentation_command(tool: DocumentationTool, arguments: &[String], path: &OsStr) -> Command { + let mut command = Command::new(tool.program()); + command + .args(arguments) + .stdin(Stdio::null()) + .env_clear() + .env("PATH", path) + .env("LC_ALL", "C"); + command +} + +#[cfg(test)] +#[path = "execution/tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "execution/finish_tests.rs"] +mod finish_tests; diff --git a/xtask/src/documentation_integrity/execution/corpus_guard.rs b/xtask/src/documentation_integrity/execution/corpus_guard.rs new file mode 100644 index 0000000..40e25e8 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/corpus_guard.rs @@ -0,0 +1,89 @@ +//! This module owns corpus revalidation around external tool execution. + +use crate::bounded_process::ProcessOutput; +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::RepositoryProcessDirectory; +use crate::repository_file::RepositoryRoot; + +use super::snapshot::DocumentationSnapshot; +use super::{DirectoryToolRunner, DocumentationTool, ToolRunner}; + +/// Executes documentation tools against one admitted immutable source snapshot. +/// +/// Construction copies bounded source bytes from retained descriptors and +/// materializes the reviewed repository namespace in a private temporary +/// directory. Every capture revalidates current source identities and exact Git +/// corpus membership before and after the inner runner executes from that +/// snapshot. Post-execution corpus drift takes precedence over a tool result. +/// +/// Construction and capture perform bounded Git child-process and repository +/// I/O. The snapshot is verification evidence only and has no durability role; +/// callers must invoke [`Self::close`] to remove it explicitly. +pub(super) struct CorpusGuardedRunner<'a, Runner> { + corpora: &'a [&'a SourceCorpus], + inner: Runner, + process_directory: &'a RepositoryProcessDirectory, + repository_root: &'a RepositoryRoot, + snapshot: DocumentationSnapshot, +} + +impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { + /// Builds the source snapshot and binds `inner` to its execution authority. + /// + /// The call allocates bounded path and source inventories, copies admitted + /// bytes, verifies fixed configuration and corpus identities, and returns + /// the exact [`DocumentationError`] from any Git, filesystem, admission, or + /// snapshot failure. + pub(super) fn new( + inner: Runner, + process_directory: &'a RepositoryProcessDirectory, + repository_root: &'a RepositoryRoot, + corpora: &'a [&'a SourceCorpus], + ) -> Result { + let snapshot = DocumentationSnapshot::create(repository_root, process_directory, corpora)?; + Ok(Self { + corpora, + inner, + process_directory, + repository_root, + snapshot, + }) + } + + /// Releases snapshot descriptors and removes the exact temporary tree. + /// + /// Removal failure is returned as [`DocumentationError::Snapshot`]. This + /// cleanup boundary does not mutate the source repository. + pub(super) fn close(self) -> Result<(), DocumentationError> { + self.snapshot.close() + } + + fn verify(&self) -> Result<(), DocumentationError> { + for corpus in self.corpora { + corpus.verify_unchanged(self.repository_root, self.process_directory)?; + } + Ok(()) + } +} + +impl ToolRunner for CorpusGuardedRunner<'_, Runner> +where + Runner: DirectoryToolRunner, +{ + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + self.verify()?; + let result = self.inner.capture_in( + self.snapshot.repository_root(), + self.snapshot.process_directory(), + tool, + arguments, + ); + self.verify()?; + result + } +} diff --git a/xtask/src/documentation_integrity/execution/finish_tests.rs b/xtask/src/documentation_integrity/execution/finish_tests.rs new file mode 100644 index 0000000..b94cfdf --- /dev/null +++ b/xtask/src/documentation_integrity/execution/finish_tests.rs @@ -0,0 +1,43 @@ +//! This module owns documentation-run closure regression evidence. + +use std::io; + +use crate::documentation_integrity::DocumentationError; + +use super::finish_run; + +#[test] +fn tool_and_snapshot_cleanup_failures_are_both_reported() { + let result = finish_run( + Err(DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + stdout: String::from("tool failure"), + stderr: String::new(), + }), + Err(DocumentationError::Snapshot { + action: "remove documentation snapshot", + source: io::Error::other("cleanup failure"), + }), + ); + + assert!(matches!( + result, + Err(DocumentationError::CheckFailures { first, second }) + if matches!( + *first, + DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + ref stdout, + ref stderr, + } if stdout == "tool failure" && stderr.is_empty() + ) && matches!( + *second, + DocumentationError::Snapshot { + action: "remove documentation snapshot", + .. + } + ) + )); +} diff --git a/xtask/src/documentation_integrity/execution/refusal_check.rs b/xtask/src/documentation_integrity/execution/refusal_check.rs new file mode 100644 index 0000000..c098086 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/refusal_check.rs @@ -0,0 +1,177 @@ +//! This module owns executable pinned-tool malformed-input refusal evidence. + +use std::fs; +use std::io; + +use crate::documentation_integrity::DocumentationError; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::{DocumentationTool, ExternalToolRunner}; + +/// Requires exact pinned-tool refusals for every malformed-input scenario. +pub(super) fn check() -> Result<(), DocumentationError> { + broken_internal_fragment_is_refused()?; + invalid_workflow_is_refused() +} + +fn broken_internal_fragment_is_refused() -> Result<(), DocumentationError> { + let directory = TestDirectory::create("broken-fragment") + .map_err(|source| fixture_error("create broken-fragment directory", source))?; + fs::write( + directory.path().join("source.md"), + "# Source\n\n[Missing](target.md#missing-heading)\n", + ) + .map_err(|source| fixture_error("write broken-fragment source", source))?; + fs::write(directory.path().join("target.md"), "# Present heading\n") + .map_err(|source| fixture_error("write broken-fragment target", source))?; + let repository_root = RepositoryRoot::open(directory.path()) + .map_err(|source| fixture_error("open broken-fragment repository", source))?; + let process_directory = repository_root + .process_directory() + .map_err(|source| fixture_error("open broken-fragment process directory", source))?; + let refusal = { + let mut runner = ExternalToolRunner { + process_directory: &process_directory, + }; + super::admit_version(&mut runner, DocumentationTool::Markdownlint)?; + super::admit_version(&mut runner, DocumentationTool::Lychee)?; + super::run_check( + &mut runner, + DocumentationTool::Markdownlint, + &[String::from("source.md"), String::from("target.md")], + )?; + super::run_check( + &mut runner, + DocumentationTool::Lychee, + &[String::from("source.md"), String::from("target.md")], + ) + }; + require_refusal( + refusal, + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + )?; + directory + .close() + .map_err(|source| fixture_error("remove broken-fragment directory", source))?; + Ok(()) +} + +fn invalid_workflow_is_refused() -> Result<(), DocumentationError> { + let directory = TestDirectory::create("invalid-workflow") + .map_err(|source| fixture_error("create invalid-workflow directory", source))?; + fs::create_dir_all(directory.path().join(".github/workflows")) + .map_err(|source| fixture_error("create invalid workflow directory", source))?; + fs::write( + directory.path().join(".github/workflows/invalid.yml"), + "name: Invalid\non: [push\n", + ) + .map_err(|source| fixture_error("write invalid workflow", source))?; + let repository_root = RepositoryRoot::open(directory.path()) + .map_err(|source| fixture_error("open invalid-workflow repository", source))?; + let process_directory = repository_root + .process_directory() + .map_err(|source| fixture_error("open invalid-workflow process directory", source))?; + let refusal = { + let mut runner = ExternalToolRunner { + process_directory: &process_directory, + }; + super::admit_version(&mut runner, DocumentationTool::Actionlint)?; + super::run_check( + &mut runner, + DocumentationTool::Actionlint, + &[String::from(".github/workflows/invalid.yml")], + ) + }; + require_refusal( + refusal, + "invalid workflow", + "actionlint", + Some(1), + "could not parse as YAML", + )?; + directory + .close() + .map_err(|source| fixture_error("remove invalid-workflow directory", source))?; + Ok(()) +} + +fn require_refusal( + refusal: Result<(), DocumentationError>, + scenario: &'static str, + program: &'static str, + code: Option, + diagnostic: &str, +) -> Result<(), DocumentationError> { + match refusal { + Err(DocumentationError::ToolFailed { + program: observed_program, + code: observed_code, + stdout, + stderr, + }) if observed_program == program + && observed_code == code + && format!("{stdout}\n{stderr}").contains(diagnostic) => + { + Ok(()) + } + Err(observed) => Err(DocumentationError::RefusalMismatch { + scenario, + observed: Some(Box::new(observed)), + }), + Ok(()) => Err(DocumentationError::RefusalMismatch { + scenario, + observed: None, + }), + } +} + +const fn fixture_error(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::RefusalFixture { action, source } +} + +#[cfg(test)] +mod tests { + use super::{DocumentationError, require_refusal}; + + #[test] + fn exact_tool_failure_is_executable_refusal_evidence() { + let refusal = Err(DocumentationError::ToolFailed { + program: "lychee", + code: Some(2), + stdout: String::from("Cannot find fragment"), + stderr: String::new(), + }); + + assert!( + require_refusal( + refusal, + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + ) + .is_ok() + ); + } + + #[test] + fn successful_malformed_input_is_a_typed_evidence_failure() { + assert!(matches!( + require_refusal( + Ok(()), + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + ), + Err(DocumentationError::RefusalMismatch { + scenario: "broken internal fragment", + observed: None, + }) + )); + } +} diff --git a/xtask/src/documentation_integrity/execution/snapshot.rs b/xtask/src/documentation_integrity/execution/snapshot.rs new file mode 100644 index 0000000..1a4987c --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot.rs @@ -0,0 +1,211 @@ +//! This module owns immutable documentation-tool input snapshots. + +mod namespace; + +use std::collections::BTreeSet; +use std::fs::{self, DirBuilder, OpenOptions}; +use std::io::{self, Write}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::documentation_integrity::error::DocumentationError; +use crate::documentation_integrity::repository_text::{self, RepositoryText}; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +const CREATION_ATTEMPTS: u16 = 1_024; +const MARKDOWNLINT_CONFIG: &str = ".markdownlint-cli2.yaml"; +static NEXT_SNAPSHOT: AtomicU64 = AtomicU64::new(0); + +/// Private repository-shaped inputs and process authority for documentation tools. +/// +/// Selected sources and the Markdown configuration contain the exact admitted +/// bytes. Other representable present regular files are copied exactly through +/// descriptor-bound, identity-checked reads so offline link validation observes +/// faithful target bytes and file types. The owned directory has no durability +/// role and is removed explicitly. +pub(super) struct DocumentationSnapshot { + directory: SnapshotDirectory, + process_directory: RepositoryProcessDirectory, + repository_root: RepositoryRoot, +} + +impl DocumentationSnapshot { + /// Materializes and revalidates one snapshot from the retained repository. + /// + /// The operation allocates bounded deterministic inventories, performs + /// filesystem writes, and starts bounded Git inventory children. It returns + /// typed corpus, policy-file, Git, or snapshot I/O failures. + pub(super) fn create( + source_root: &RepositoryRoot, + source_process_directory: &RepositoryProcessDirectory, + corpora: &[&SourceCorpus], + ) -> Result { + let config = repository_text::read(source_root, MARKDOWNLINT_CONFIG)?; + let directory = SnapshotDirectory::create()?; + let materialized = materialize(directory.path(), corpora, &config)?; + namespace::materialize( + directory.path(), + source_root, + source_process_directory, + &materialized, + )?; + verify_sources(source_root, source_process_directory, corpora, &config)?; + let repository_root = open_snapshot_root(directory.path())?; + let process_directory = repository_root.process_directory().map_err(|source| { + snapshot_io("open documentation snapshot process directory", source) + })?; + Ok(Self { + directory, + process_directory, + repository_root, + }) + } + + /// Returns the descriptor-backed child working directory for the snapshot. + pub(super) const fn process_directory(&self) -> &RepositoryProcessDirectory { + &self.process_directory + } + + /// Returns the capability-relative root used by in-process snapshot readers. + pub(super) const fn repository_root(&self) -> &RepositoryRoot { + &self.repository_root + } + + /// Closes owned directory descriptors and removes the exact snapshot tree. + /// + /// Cleanup does not mutate the source repository and reports removal failure + /// through [`DocumentationError::Snapshot`]. + pub(super) fn close(self) -> Result<(), DocumentationError> { + let Self { + directory, + process_directory, + repository_root, + } = self; + drop(process_directory); + drop(repository_root); + directory + .close() + .map_err(|source| snapshot_io("remove documentation snapshot", source)) + } +} + +fn materialize( + destination: &Path, + corpora: &[&SourceCorpus], + config: &RepositoryText, +) -> Result, DocumentationError> { + let mut materialized = BTreeSet::new(); + for corpus in corpora { + corpus.materialize(destination)?; + for path in corpus.paths() { + materialized.insert(PathBuf::from(path)); + } + } + write_config(destination, config)?; + materialized.insert(PathBuf::from(MARKDOWNLINT_CONFIG)); + Ok(materialized) +} + +fn write_config(destination: &Path, config: &RepositoryText) -> Result<(), DocumentationError> { + let path = destination.join(MARKDOWNLINT_CONFIG); + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(path) + .map_err(|source| snapshot_io("create documentation snapshot configuration", source))?; + output + .write_all(config.as_str().as_bytes()) + .map_err(|source| snapshot_io("write documentation snapshot configuration", source)) +} + +fn verify_sources( + source_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + corpora: &[&SourceCorpus], + config: &RepositoryText, +) -> Result<(), DocumentationError> { + for corpus in corpora { + corpus.verify_unchanged(source_root, process_directory)?; + } + config.verify(source_root) +} + +fn open_snapshot_root(path: &Path) -> Result { + RepositoryRoot::open(path) + .map_err(|source| snapshot_io("open documentation snapshot root", source)) +} + +struct SnapshotDirectory { + path: PathBuf, + active: bool, +} + +impl SnapshotDirectory { + fn create() -> Result { + for _ in 0_u16..CREATION_ATTEMPTS { + let sequence = next_sequence()?; + let path = std::env::temp_dir().join(format!( + "keep-documentation-snapshot-{}-{sequence}", + std::process::id() + )); + let mut builder = DirBuilder::new(); + builder.mode(0o700); + match builder.create(&path) { + Ok(()) => return Ok(Self { path, active: true }), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(snapshot_io("create documentation snapshot", source)); + } + } + } + Err(snapshot_io( + "create documentation snapshot", + io::Error::new( + io::ErrorKind::AlreadyExists, + "documentation snapshot collision bound exhausted", + ), + )) + } + + fn path(&self) -> &Path { + self.path.as_path() + } + + fn close(mut self) -> Result<(), io::Error> { + fs::remove_dir_all(&self.path)?; + self.active = false; + Ok(()) + } +} + +impl Drop for SnapshotDirectory { + fn drop(&mut self) { + if self.active { + drop(fs::remove_dir_all(&self.path)); + } + } +} + +fn next_sequence() -> Result { + NEXT_SNAPSHOT + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(1) + }) + .map_err(|_| { + snapshot_io( + "allocate documentation snapshot identity", + io::Error::other("documentation snapshot sequence exhausted"), + ) + }) +} + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} + +#[cfg(test)] +#[path = "snapshot/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/execution/snapshot/namespace.rs b/xtask/src/documentation_integrity/execution/snapshot/namespace.rs new file mode 100644 index 0000000..4295f34 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot/namespace.rs @@ -0,0 +1,182 @@ +//! This module owns faithful non-corpus snapshot namespace materialization. + +use std::collections::BTreeSet; +use std::fs::{self, File, OpenOptions}; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use xtask::protocol_admission::posix_relative_path; + +use crate::documentation_integrity::error::DocumentationError; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::{ + OpenRepositoryFileError, RepositoryFileIdentity, RepositoryProcessDirectory, RepositoryRoot, + copy_exact, +}; + +const CORPUS: &str = "documentation snapshot namespace"; +const FILE_LIMIT_BYTES: u64 = 4 * 1_024 * 1_024; +const NAMESPACE_DELETED: [&str; 3] = ["ls-files", "-z", "--deleted"]; +const NAMESPACE_PRESENT: [&str; 5] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", +]; +const TOTAL_LIMIT_BYTES: u64 = 64 * 1_024 * 1_024; + +pub(super) fn materialize( + destination: &Path, + source_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + materialized: &BTreeSet, +) -> Result<(), DocumentationError> { + let present = paths_with( + &NAMESPACE_PRESENT, + "git documentation snapshot present paths", + |command| process_directory.spawn(command), + )?; + let deleted = paths_with( + &NAMESPACE_DELETED, + "git documentation snapshot deleted paths", + |command| process_directory.spawn(command), + )?; + let mut total = 0_u64; + for path in present.difference(&deleted) { + if let Some((relative, text)) = snapshot_relative(path)? + && !materialized.contains(&relative) + { + total = materialize_file(destination, source_root, &relative, &text, total)?; + } + } + Ok(()) +} + +fn materialize_file( + destination: &Path, + source_root: &RepositoryRoot, + relative: &Path, + path: &str, + total: u64, +) -> Result { + let file = open_source(source_root, relative, path)?; + let admitted = identity(&file, path)?; + refuse_file_bound(path, admitted.bytes())?; + let next_total = total + .checked_add(admitted.bytes()) + .ok_or(DocumentationError::CorpusSizeOverflow(CORPUS))?; + refuse_total_bound(next_total)?; + let mut output = create_destination(destination, relative)?; + copy_exact(&file, &mut output, admitted.bytes()) + .map_err(|source| snapshot_io("copy documentation snapshot namespace file", source))?; + verify_identity(&file, source_root, relative, path, &admitted)?; + Ok(next_total) +} + +fn open_source( + source_root: &RepositoryRoot, + relative: &Path, + path: &str, +) -> Result { + match source_root.open_file(relative) { + Ok(file) => Ok(file), + Err(OpenRepositoryFileError::Io(source)) => Err(DocumentationError::Inspect { + corpus: CORPUS, + path: path.to_owned(), + source, + }), + Err(OpenRepositoryFileError::NonRegular) => Err(DocumentationError::NonRegular { + corpus: CORPUS, + path: path.to_owned(), + }), + } +} + +fn create_destination(destination: &Path, relative: &Path) -> Result { + let path = destination.join(relative); + let parent = path.parent().ok_or_else(|| { + snapshot_io( + "resolve documentation snapshot namespace parent", + io::Error::other("namespace path has no parent"), + ) + })?; + fs::create_dir_all(parent) + .map_err(|source| snapshot_io("create documentation snapshot namespace", source))?; + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(path) + .map_err(|source| snapshot_io("create documentation snapshot namespace file", source)) +} + +fn verify_identity( + retained: &File, + source_root: &RepositoryRoot, + relative: &Path, + path: &str, + admitted: &RepositoryFileIdentity, +) -> Result<(), DocumentationError> { + let retained_identity = identity(retained, path)?; + let current = open_source(source_root, relative, path)?; + let current_identity = identity(¤t, path)?; + if &retained_identity == admitted && ¤t_identity == admitted { + Ok(()) + } else { + Err(DocumentationError::CorpusChanged { + corpus: CORPUS, + path: path.to_owned(), + }) + } +} + +fn identity(file: &File, path: &str) -> Result { + RepositoryFileIdentity::read(file).map_err(|source| DocumentationError::Inspect { + corpus: CORPUS, + path: path.to_owned(), + source, + }) +} + +fn refuse_file_bound(path: &str, observed: u64) -> Result<(), DocumentationError> { + if observed <= FILE_LIMIT_BYTES { + Ok(()) + } else { + Err(DocumentationError::CorpusFileTooLarge { + corpus: CORPUS, + path: path.to_owned(), + maximum: FILE_LIMIT_BYTES, + observed, + }) + } +} + +const fn refuse_total_bound(observed: u64) -> Result<(), DocumentationError> { + if observed <= TOTAL_LIMIT_BYTES { + Ok(()) + } else { + Err(DocumentationError::CorpusTooLarge { + corpus: CORPUS, + maximum: TOTAL_LIMIT_BYTES, + observed, + }) + } +} + +fn snapshot_relative(path: &GitPath) -> Result, DocumentationError> { + let Ok(text) = std::str::from_utf8(path.as_bytes()) else { + return Ok(None); + }; + posix_relative_path(text) + .map(|relative| Some((relative, text.to_owned()))) + .map_err(|_| DocumentationError::InvalidPath { + corpus: CORPUS, + path: text.to_owned(), + }) +} + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} diff --git a/xtask/src/documentation_integrity/execution/snapshot/tests.rs b/xtask/src/documentation_integrity/execution/snapshot/tests.rs new file mode 100644 index 0000000..3f3cfad --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot/tests.rs @@ -0,0 +1,101 @@ +//! This module owns documentation snapshot namespace regression evidence. + +use std::error::Error; +use std::fs; +use std::io::{self, Read}; +use std::os::unix::fs::symlink; +use std::path::Path; + +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +use super::DocumentationSnapshot; + +#[test] +fn non_markdown_link_target_preserves_admitted_bytes() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-link-target")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::create_dir(root.join("docs"))?; + fs::write( + root.join("docs/README.md"), + "[target](../target.html#section)\n", + )?; + fs::write(root.join("target.html"), "

Target

\n")?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; + run_git( + root, + &[ + "add", + "--", + ".markdownlint-cli2.yaml", + "docs/README.md", + "target.html", + ], + )?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let markdown = SourceCorpus::markdown(&repository_root, &process_directory)?; + let snapshot = + DocumentationSnapshot::create(&repository_root, &process_directory, &[&markdown])?; + let mut target = snapshot + .repository_root() + .open_file(Path::new("target.html")) + .map_err(|_| io::Error::other("open snapshot target"))?; + let mut observed = Vec::new(); + target.read_to_end(&mut observed)?; + + assert_eq!(observed, b"

Target

\n"); + drop(target); + snapshot.close()?; + drop(markdown); + directory.close()?; + Ok(()) +} + +#[test] +fn nonregular_link_target_is_refused_before_validation() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-symlink-target")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::create_dir(root.join("docs"))?; + fs::write(root.join("docs/README.md"), "[target](../target.html)\n")?; + fs::write(root.join("real.html"), "

Target

\n")?; + symlink("real.html", root.join("target.html"))?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; + run_git( + root, + &[ + "add", + "--", + ".markdownlint-cli2.yaml", + "docs/README.md", + "real.html", + "target.html", + ], + )?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let markdown = SourceCorpus::markdown(&repository_root, &process_directory)?; + + let result = DocumentationSnapshot::create(&repository_root, &process_directory, &[&markdown]); + + assert!(matches!( + result, + Err(crate::documentation_integrity::DocumentationError::NonRegular { + corpus: "documentation snapshot namespace", + ref path, + }) if path == "target.html" + )); + drop(markdown); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs new file mode 100644 index 0000000..3f68f85 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -0,0 +1,290 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::ffi::OsString; +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use crate::bounded_process::ProcessOutput; +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +use super::corpus_guard::CorpusGuardedRunner; +use super::{ + DirectoryToolRunner, DocumentationError, DocumentationTool, ToolRunner, documentation_command, +}; + +struct RecordingRunner { + calls: Vec<(DocumentationTool, Vec)>, + outputs: VecDeque, +} + +struct ReplacingRunner { + repository: PathBuf, +} + +#[test] +fn documentation_tools_receive_only_reviewed_environment() { + let path = OsString::from("/reviewed/tools"); + let command = documentation_command(DocumentationTool::Markdownlint, &[], &path); + let observed = command + .get_envs() + .map(|(name, value)| (name.to_owned(), value.map(OsString::from))) + .collect::>(); + let expected = BTreeMap::from([ + (OsString::from("LC_ALL"), Some(OsString::from("C"))), + (OsString::from("PATH"), Some(path)), + ]); + + assert_eq!(observed, expected); + assert_eq!( + DocumentationError::EnvironmentUnavailable("PATH").to_string(), + "documentation tool environment variable `PATH` is unavailable" + ); +} + +#[test] +fn admitted_tools_run_with_exact_arguments_and_silent_success() { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + success(), + success(), + version(DocumentationTool::Actionlint), + success(), + ]); + let markdown = [String::from("README.md")]; + let workflows = [String::from(".github/workflows/ci.yml")]; + + assert!(super::run_with(&mut runner, &markdown, &workflows).is_ok()); + assert_eq!(runner.calls.len(), 6); + assert_eq!( + runner.calls.get(2), + Some(&( + DocumentationTool::Markdownlint, + vec![ + String::from("--no-globs"), + String::from("--"), + String::from("README.md") + ] + )) + ); + assert_eq!( + runner.calls.get(5), + Some(&( + DocumentationTool::Actionlint, + vec![ + String::from("-shellcheck="), + String::from("-pyflakes="), + String::from(".github/workflows/ci.yml") + ] + )) + ); +} + +#[test] +fn link_check_runs_after_markdownlint_returns_nonzero() { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + failure(b"lint", b""), + success(), + ]); + + let result = super::run_with(&mut runner, &[String::from("README.md")], &[]); + + assert!(matches!( + result, + Err(DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + ref stdout, + ref stderr, + }) if stdout == "lint" && stderr.is_empty() + )); + assert_eq!(runner.calls.len(), 4); + assert_eq!( + runner.calls.get(3).map(|call| call.0), + Some(DocumentationTool::Lychee) + ); +} + +#[test] +fn simultaneous_markdown_failures_are_both_reported() -> Result<(), &'static str> { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + failure(b"lint refusal", b""), + failure(b"link refusal", b""), + ]); + + let result = super::run_with(&mut runner, &[String::from("README.md")], &[]); + let diagnostic = result + .err() + .ok_or("both Markdown checks unexpectedly succeeded")? + .to_string(); + + assert!(diagnostic.contains("lint refusal")); + assert!(diagnostic.contains("link refusal")); + Ok(()) +} + +#[test] +fn unreviewed_version_stops_before_tool_execution() { + let mut runner = RecordingRunner::new([ProcessOutput { + code: Some(0), + succeeded: true, + stdout: b"markdownlint-cli2 v999.0.0\n".to_vec(), + stderr: Vec::new(), + }]); + + let result = super::run_with(&mut runner, &[], &[]); + + assert!(matches!( + result, + Err(DocumentationError::VersionMismatch { + program: "markdownlint-cli2", + expected: "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + ref observed, + }) if observed == "markdownlint-cli2 v999.0.0" + )); + assert_eq!(runner.calls.len(), 1); +} + +#[test] +fn corpus_guard_executes_against_the_admitted_source_snapshot() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-transient-source")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::create_dir(root.join("docs"))?; + fs::write(root.join("docs/selected.md"), "# Original\n")?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + let corpora = [&corpus]; + let replacing = ReplacingRunner { + repository: root.to_owned(), + }; + let mut runner = + CorpusGuardedRunner::new(replacing, &process_directory, &repository_root, &corpora)?; + + let output = runner.capture(DocumentationTool::Markdownlint, &[])?; + + assert_eq!(output.stdout, b"# Original\n"); + runner.close()?; + drop(corpus); + directory.close()?; + Ok(()) +} + +impl RecordingRunner { + fn new(outputs: impl IntoIterator) -> Self { + Self { + calls: Vec::new(), + outputs: outputs.into_iter().collect(), + } + } +} + +impl ToolRunner for RecordingRunner { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + self.calls.push((tool, arguments.to_vec())); + self.outputs + .pop_front() + .ok_or(DocumentationError::RepositoryContract { + path: "test runner", + requirement: "one output exists for every expected call", + }) + } +} + +impl DirectoryToolRunner for ReplacingRunner { + fn capture_in( + &mut self, + execution_root: &RepositoryRoot, + _process_directory: &crate::repository_file::RepositoryProcessDirectory, + _tool: DocumentationTool, + _arguments: &[String], + ) -> Result { + let selected = self.repository.join("docs"); + let retained = self.repository.join("retained-docs"); + fs::rename(&selected, &retained) + .map_err(|source| fixture_io("retain selected directory", source))?; + fs::create_dir(&selected) + .map_err(|source| fixture_io("create substitute directory", source))?; + fs::write(selected.join("selected.md"), "# Substitute\n") + .map_err(|source| fixture_io("write substitute source", source))?; + let observed = read_source(execution_root, Path::new("docs/selected.md"))?; + fs::remove_dir_all(&selected) + .map_err(|source| fixture_io("remove substitute directory", source))?; + fs::rename(&retained, &selected) + .map_err(|source| fixture_io("restore selected directory", source))?; + Ok(ProcessOutput { + code: Some(0), + succeeded: true, + stdout: observed, + stderr: Vec::new(), + }) + } +} + +fn read_source( + repository_root: &RepositoryRoot, + relative: &Path, +) -> Result, DocumentationError> { + let mut file = repository_root.open_file(relative).map_err(|_| { + fixture_io( + "open execution source", + std::io::Error::other("open failed"), + ) + })?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|source| fixture_io("read execution source", source))?; + Ok(bytes) +} + +fn fixture_io(requirement: &'static str, source: std::io::Error) -> DocumentationError { + DocumentationError::Inspect { + corpus: "test", + path: requirement.to_owned(), + source, + } +} + +fn version(tool: DocumentationTool) -> ProcessOutput { + ProcessOutput { + code: Some(0), + succeeded: true, + stdout: format!("{}\n", tool.expected_version()).into_bytes(), + stderr: Vec::new(), + } +} + +const fn success() -> ProcessOutput { + ProcessOutput { + code: Some(0), + succeeded: true, + stdout: Vec::new(), + stderr: Vec::new(), + } +} + +fn failure(stdout: &[u8], stderr: &[u8]) -> ProcessOutput { + ProcessOutput { + code: Some(1), + succeeded: false, + stdout: stdout.to_vec(), + stderr: stderr.to_vec(), + } +} diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs new file mode 100644 index 0000000..140a925 --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -0,0 +1,179 @@ +//! This module owns the committed Node documentation-tool graph contract. + +mod unique_json; + +use serde_json::{Map, Value}; + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text::{self, RepositoryText}; + +const INSTALLER_PATH: &str = "scripts/install_documentation_tools.sh"; +const INSTALLER_DIGEST: [u8; 32] = [ + 0x12, 0xfb, 0x82, 0xcd, 0xdb, 0x65, 0x52, 0xe5, 0xae, 0xdb, 0x14, 0x54, 0x83, 0xf4, 0x8a, 0x8a, + 0x8b, 0x35, 0x54, 0xea, 0x2a, 0x90, 0xeb, 0xd4, 0x82, 0xd7, 0x4a, 0x61, 0x1f, 0xf6, 0xd3, 0xfd, +]; +const LOCK_DIGEST: [u8; 32] = [ + 0x74, 0x21, 0xce, 0x90, 0xdd, 0x52, 0x33, 0xfe, 0x99, 0x1a, 0x0b, 0x7e, 0xdd, 0xaa, 0xb7, 0x53, + 0x63, 0xf3, 0xad, 0x3b, 0x0f, 0x9e, 0x7d, 0xa2, 0xa4, 0x77, 0x65, 0xf0, 0x9c, 0x1d, 0xcb, 0x3b, +]; +const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; +const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; + +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result<[RepositoryText; 3], DocumentationError> { + let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; + let lock = repository_text::read(repository_root, LOCK_PATH)?; + let installer = repository_text::read(repository_root, INSTALLER_PATH)?; + admit(manifest.as_str(), lock.as_str(), installer.as_str())?; + admit_lock_bytes(lock.as_str())?; + manifest.verify(repository_root)?; + lock.verify(repository_root)?; + installer.verify(repository_root)?; + Ok([manifest, lock, installer]) +} + +fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), DocumentationError> { + let manifest = parse(MANIFEST_PATH, manifest)?; + admit_manifest(&manifest)?; + let lock = parse(LOCK_PATH, lock)?; + admit_lock(&lock)?; + admit_installer(installer) +} + +fn admit_manifest(manifest: &Value) -> Result<(), DocumentationError> { + if manifest.get("overrides").is_some() { + return Err(contract(MANIFEST_PATH, "dependency overrides are absent")); + } + let observed = manifest + .get("dependencies") + .and_then(|dependencies| dependencies.get("markdownlint-cli2")) + .and_then(Value::as_str); + if observed == Some("0.23.2") { + Ok(()) + } else { + Err(DocumentationError::RepositoryValue { + path: MANIFEST_PATH, + field: "dependencies.markdownlint-cli2", + expected: "0.23.2", + observed: observed.map(str::to_owned), + }) + } +} + +fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { + if lock.get("lockfileVersion").and_then(Value::as_u64) != Some(3) { + return Err(contract(LOCK_PATH, "lockfileVersion is exactly 3")); + } + let packages = lock.get("packages").and_then(Value::as_object).ok_or( + DocumentationError::RepositoryContract { + path: LOCK_PATH, + requirement: "packages is an object", + }, + )?; + require_package_value( + packages, + "", + &["dependencies", "markdownlint-cli2"], + "packages[\"\"].dependencies.markdownlint-cli2", + "0.23.2", + )?; + require_package_value( + packages, + "node_modules/markdownlint-cli2", + &["dependencies", "js-yaml"], + "packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml", + "5.2.2", + )?; + require_package_value( + packages, + "node_modules/js-yaml", + &["version"], + "packages[\"node_modules/js-yaml\"].version", + "5.2.2", + )?; + require_package_value( + packages, + "node_modules/markdown-it", + &["version"], + "packages[\"node_modules/markdown-it\"].version", + "14.3.0", + )?; + require_provenance(packages)?; + Ok(()) +} + +fn admit_installer(installer: &str) -> Result<(), DocumentationError> { + if blake3::hash(installer.as_bytes()).as_bytes() == &INSTALLER_DIGEST { + Ok(()) + } else { + Err(contract( + INSTALLER_PATH, + "installer bytes match the reviewed digest", + )) + } +} + +fn admit_lock_bytes(lock: &str) -> Result<(), DocumentationError> { + if blake3::hash(lock.as_bytes()).as_bytes() == &LOCK_DIGEST { + Ok(()) + } else { + Err(contract(LOCK_PATH, "lock bytes match the reviewed digest")) + } +} + +fn parse(path: &'static str, raw: &str) -> Result { + unique_json::parse(raw).map_err(|source| DocumentationError::RepositoryJson { path, source }) +} + +fn require_package_value( + packages: &Map, + package: &'static str, + fields: &[&str], + field: &'static str, + expected: &'static str, +) -> Result<(), DocumentationError> { + let mut value = packages.get(package); + for field in fields { + value = value.and_then(|current| current.get(field)); + } + let observed = value.and_then(Value::as_str); + if observed == Some(expected) { + Ok(()) + } else { + Err(DocumentationError::RepositoryValue { + path: LOCK_PATH, + field, + expected, + observed: observed.map(str::to_owned), + }) + } +} + +fn require_provenance(packages: &Map) -> Result<(), DocumentationError> { + for (path, package) in packages { + if !path.is_empty() { + let object = package.as_object(); + if !object.is_some_and(|fields| { + fields.contains_key("resolved") && fields.contains_key("integrity") + }) { + return Err(DocumentationError::RepositoryContractAt { + path: LOCK_PATH, + subject: path.clone(), + requirement: "package records resolved and integrity fields", + }); + } + } + } + Ok(()) +} + +const fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { path, requirement } +} + +#[cfg(test)] +#[path = "node_toolchain/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs new file mode 100644 index 0000000..1336533 --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -0,0 +1,197 @@ +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +#[path = "tests/lock_graph.rs"] +mod lock_graph; + +const MANIFEST: &str = r#"{"dependencies":{"markdownlint-cli2":"0.23.2"}}"#; +const LOCK: &str = r#"{ + "lockfileVersion": 3, + "packages": { + "": {"dependencies": {"markdownlint-cli2": "0.23.2"}}, + "node_modules/markdownlint-cli2": { + "dependencies": {"js-yaml": "5.2.2"}, + "resolved": "example", + "integrity": "example" + }, + "node_modules/js-yaml": { + "version": "5.2.2", + "resolved": "example", + "integrity": "example" + }, + "node_modules/markdown-it": { + "version": "14.3.0", + "resolved": "example", + "integrity": "example" + } + } +}"#; +const INSTALLER: &str = include_str!("../../../../scripts/install_documentation_tools.sh"); +const REVIEWED_NPM_CI: &str = concat!( + "npm ci \\\n", + " --prefix \"$npm_dir\" \\\n", + " --ignore-scripts \\\n", + " --no-audit \\\n", + " --no-fund\n", +); + +#[test] +fn admitted_node_toolchain_is_exact_and_lockfile_installed() { + assert!(super::admit(MANIFEST, LOCK, INSTALLER).is_ok()); +} + +#[test] +fn dependency_overrides_are_refused() { + let manifest = r#"{"overrides":{},"dependencies":{"markdownlint-cli2":"0.23.2"}}"#; + assert!(matches!( + super::admit(manifest, LOCK, INSTALLER), + Err(super::DocumentationError::RepositoryContract { + path: super::MANIFEST_PATH, + requirement: "dependency overrides are absent", + }) + )); +} + +#[test] +fn manifest_dependency_version_drift_is_refused() { + let manifest = r#"{"dependencies":{"markdownlint-cli2":"999.0.0"}}"#; + assert!(matches!( + super::admit(manifest, LOCK, INSTALLER), + Err(super::DocumentationError::RepositoryValue { + path: super::MANIFEST_PATH, + field: "dependencies.markdownlint-cli2", + expected: "0.23.2", + observed: Some(ref observed), + }) if observed == "999.0.0" + )); +} + +#[test] +fn duplicate_object_members_are_refused_at_every_depth() { + let manifest = concat!( + r#"{"dependencies":{"markdownlint-cli2":"999.0.0"},"#, + r#""dependencies":{"markdownlint-cli2":"0.23.2"}}"#, + ); + let lock = LOCK.replacen( + r#""version": "14.3.0","#, + r#""version": "999.0.0", "version": "14.3.0","#, + 1, + ); + for result in [ + super::admit(manifest, LOCK, INSTALLER), + super::admit(MANIFEST, &lock, INSTALLER), + ] { + assert!(matches!( + result, + Err(super::DocumentationError::RepositoryJson { .. }) + )); + } +} + +#[test] +fn dependency_version_drift_is_refused() { + let lock = LOCK.replacen("\"5.2.2\"", "\"5.2.1\"", 1); + let error = super::admit(MANIFEST, &lock, INSTALLER); + assert!(matches!( + &error, + Err(super::DocumentationError::RepositoryValue { + path: super::LOCK_PATH, + field: "packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml", + expected: "5.2.2", + observed: Some(observed), + }) if observed == "5.2.1" + )); + assert_eq!( + error.as_ref().map_err(ToString::to_string), + Err(String::from(concat!( + "repository file `scripts/documentation-tools/package-lock.json` requires ", + "`packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml` to be ", + "\"5.2.2\"; observed \"5.2.1\"" + ))) + ); +} + +#[test] +fn missing_dependency_coordinate_is_refused_precisely() { + let lock = LOCK.replacen("\"version\": \"14.3.0\"", "\"missing\": \"14.3.0\"", 1); + assert!(matches!( + super::admit(MANIFEST, &lock, INSTALLER), + Err(super::DocumentationError::RepositoryValue { + path: super::LOCK_PATH, + field: "packages[\"node_modules/markdown-it\"].version", + expected: "14.3.0", + observed: None, + }) + )); +} + +#[test] +fn missing_package_provenance_is_refused() { + let lock = LOCK.replacen("\"integrity\": \"example\"", "\"missing\": \"example\"", 1); + let error = super::admit(MANIFEST, &lock, INSTALLER); + assert!(matches!( + &error, + Err(super::DocumentationError::RepositoryContractAt { + path: super::LOCK_PATH, + subject, + requirement: "package records resolved and integrity fields", + }) if subject == "node_modules/markdownlint-cli2" + )); + let diagnostic = error.as_ref().map_err(ToString::to_string); + assert!(diagnostic.is_err_and(|message| message.contains("node_modules/markdownlint-cli2"))); +} + +#[test] +fn unlocked_installer_is_refused() { + assert!(matches!( + super::admit(MANIFEST, LOCK, "npm install markdownlint-cli2\n"), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installer bytes match the reviewed digest", + }) + )); +} + +#[test] +fn comments_cannot_mask_an_unlocked_installer() { + let installer = "# npm ci\n# package-lock.json\nnpm install markdownlint-cli2\n"; + assert!(matches!( + super::admit(MANIFEST, LOCK, installer), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installer bytes match the reviewed digest", + }) + )); +} + +#[test] +fn dead_shell_structure_cannot_mask_an_unreviewed_command() { + let installer = format!( + "# package-lock.json\n\ + unused() {{\n\ + \x20 cat <<'REVIEWED'\n\ + {REVIEWED_NPM_CI}\n\ + REVIEWED\n\ + }}\n\ + npm ci --prefix /tmp/unreviewed --foreground-scripts\n", + ); + assert!(matches!( + super::admit(MANIFEST, LOCK, &installer), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installer bytes match the reviewed digest", + }) + )); +} + +#[test] +fn committed_node_toolchain_satisfies_the_rust_law() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + super::check(&repository_root)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} diff --git a/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs new file mode 100644 index 0000000..3d7515e --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs @@ -0,0 +1,63 @@ +use std::fs; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::super::{DocumentationError, INSTALLER_PATH, LOCK_PATH, MANIFEST_PATH}; + +const INSTALLER: &str = include_str!("../../../../../scripts/install_documentation_tools.sh"); +const LOCK: &str = include_str!("../../../../../scripts/documentation-tools/package-lock.json"); +const MANIFEST: &str = include_str!("../../../../../scripts/documentation-tools/package.json"); + +#[test] +fn unreviewed_lock_graph_changes_are_refused() -> Result<(), Box> { + let altered = LOCK.replacen( + "\"resolved\": \"https://registry.npmjs.org/", + "\"resolved\": \"https://example.com/", + 1, + ); + let extra = LOCK.replacen( + "\n }\n}", + concat!( + ",\n", + " \"node_modules/unreviewed\": {\n", + " \"version\": \"1.0.0\",\n", + " \"resolved\": \"https://example.com/unreviewed.tgz\",\n", + " \"integrity\": \"sha512-example\"\n", + " }\n", + " }\n", + "}" + ), + 1, + ); + + for lock in [altered, extra] { + assert!(matches!( + check_with_lock(&lock)?, + Err(DocumentationError::RepositoryContract { + path: LOCK_PATH, + requirement: "lock bytes match the reviewed digest", + }) + )); + } + Ok(()) +} + +fn check_with_lock( + lock: &str, +) -> Result, Box> { + let repository = TestDirectory::create("node-lock-graph")?; + write_repository(&repository, lock)?; + let root = RepositoryRoot::open(repository.path())?; + let result = super::super::check(&root).map(drop); + repository.close()?; + Ok(result) +} + +fn write_repository(repository: &TestDirectory, lock: &str) -> Result<(), std::io::Error> { + let tool_directory = repository.path().join("scripts/documentation-tools"); + fs::create_dir_all(&tool_directory)?; + fs::write(repository.path().join(MANIFEST_PATH), MANIFEST)?; + fs::write(repository.path().join(LOCK_PATH), lock)?; + fs::write(repository.path().join(INSTALLER_PATH), INSTALLER) +} diff --git a/xtask/src/documentation_integrity/node_toolchain/unique_json.rs b/xtask/src/documentation_integrity/node_toolchain/unique_json.rs new file mode 100644 index 0000000..35d5c47 --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/unique_json.rs @@ -0,0 +1,130 @@ +//! This module owns duplicate-refusing repository JSON admission. + +use std::fmt; + +use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Number, Value}; + +pub(super) fn parse(raw: &str) -> Result { + serde_json::from_str::(raw).map(|value| value.0) +} + +struct UniqueValue(Value); + +impl<'de> Deserialize<'de> for UniqueValue { + fn deserialize( + deserializer: DeserializerType, + ) -> Result + where + DeserializerType: Deserializer<'de>, + { + deserializer.deserialize_any(UniqueVisitor) + } +} + +struct UniqueVisitor; + +impl<'de> Visitor<'de> for UniqueVisitor { + type Value = UniqueValue; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object members") + } + + fn visit_bool(self, value: bool) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Bool(value))) + } + + fn visit_i64(self, value: i64) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Number(Number::from(value)))) + } + + fn visit_u64(self, value: u64) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Number(Number::from(value)))) + } + + fn visit_f64(self, value: f64) -> Result + where + ErrorType: de::Error, + { + Number::from_f64(value) + .map(Value::Number) + .map(UniqueValue) + .ok_or_else(|| ErrorType::custom("JSON number is not finite")) + } + + fn visit_str(self, value: &str) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::String(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::String(value))) + } + + fn visit_none(self) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Null)) + } + + fn visit_unit(self) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Null)) + } + + fn visit_some( + self, + deserializer: DeserializerType, + ) -> Result + where + DeserializerType: Deserializer<'de>, + { + UniqueValue::deserialize(deserializer) + } + + fn visit_seq(self, mut sequence: Sequence) -> Result + where + Sequence: SeqAccess<'de>, + { + let mut values = Vec::new(); + while let Some(value) = sequence.next_element::()? { + values.push(value.0); + } + Ok(UniqueValue(Value::Array(values))) + } + + fn visit_map(self, mut object: Object) -> Result + where + Object: MapAccess<'de>, + { + let mut values = Map::new(); + while let Some(key) = object.next_key::()? { + if values.contains_key(&key) { + return Err(de::Error::custom(format!( + "duplicate object member {key:?}" + ))); + } + let value = object.next_value::()?; + values.insert(key, value.0); + } + Ok(UniqueValue(Value::Object(values))) + } +} diff --git a/xtask/src/documentation_integrity/policy_corpus.rs b/xtask/src/documentation_integrity/policy_corpus.rs new file mode 100644 index 0000000..3384d6c --- /dev/null +++ b/xtask/src/documentation_integrity/policy_corpus.rs @@ -0,0 +1,52 @@ +//! This module owns the retained fixed-policy identity corpus. + +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +use super::contributor_contract; +use super::dependabot; +use super::error::DocumentationError; +use super::node_toolchain; +use super::repository_text::RepositoryText; +use super::workflow_contract; + +const POLICY_SOURCE_COUNT: usize = 7; + +/// The fixed policy files whose admitted identities govern one complete check. +pub(super) struct FixedPolicyCorpus { + sources: [RepositoryText; POLICY_SOURCE_COUNT], +} + +impl FixedPolicyCorpus { + /// Admits every fixed policy file and retains its open identity witness. + pub(super) fn admit( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + let [contributing, standards] = contributor_contract::check(repository_root)?; + let [manifest, lock, installer] = node_toolchain::check(repository_root)?; + let dependabot = dependabot::check(repository_root, process_directory)?; + let workflow = workflow_contract::check(repository_root)?; + Ok(Self { + sources: [ + contributing, + standards, + manifest, + lock, + installer, + dependabot, + workflow, + ], + }) + } + + /// Revalidates every retained policy identity at the final success boundary. + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + ) -> Result<(), DocumentationError> { + for source in &self.sources { + source.verify(repository_root)?; + } + Ok(()) + } +} diff --git a/xtask/src/documentation_integrity/repository_text.rs b/xtask/src/documentation_integrity/repository_text.rs new file mode 100644 index 0000000..c9bbb8b --- /dev/null +++ b/xtask/src/documentation_integrity/repository_text.rs @@ -0,0 +1,107 @@ +//! This module owns bounded UTF-8 reads of fixed repository policy files. + +use std::io::{self, Read}; +use std::path::Path; + +use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; + +use super::error::DocumentationError; + +const MAX_REPOSITORY_FILE_BYTES: u64 = 1_048_576; + +/// Bounded UTF-8 policy text with the file identity admitted for its bytes. +pub(super) struct RepositoryText { + identity: RepositoryFileIdentity, + path: &'static str, + text: String, +} + +impl RepositoryText { + /// Returns the admitted UTF-8 policy text. + pub(super) fn as_str(&self) -> &str { + &self.text + } + + /// Revalidates that the policy path still names the admitted file identity. + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + ) -> Result<(), DocumentationError> { + let current = match repository_root.open_file(Path::new(self.path)) { + Ok(file) => file, + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(DocumentationError::RepositoryFileChanged(self.path)); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(DocumentationError::RepositoryFileInspect { + path: self.path, + source, + }); + } + Err(OpenRepositoryFileError::NonRegular) => { + return Err(DocumentationError::RepositoryFileChanged(self.path)); + } + }; + let current = RepositoryFileIdentity::read(¤t).map_err(|source| { + DocumentationError::RepositoryFileInspect { + path: self.path, + source, + } + })?; + if current == self.identity { + Ok(()) + } else { + Err(DocumentationError::RepositoryFileChanged(self.path)) + } + } +} + +/// Reads one fixed policy path through the retained repository authority. +pub(super) fn read( + repository_root: &RepositoryRoot, + path: &'static str, +) -> Result { + let file = repository_root + .open_file(Path::new(path)) + .map_err(|error| open_error(path, error))?; + let identity = RepositoryFileIdentity::read(&file) + .map_err(|source| DocumentationError::RepositoryFileInspect { path, source })?; + let read_bound = MAX_REPOSITORY_FILE_BYTES.checked_add(1).ok_or( + DocumentationError::RepositoryFileTooLarge { + path, + maximum: MAX_REPOSITORY_FILE_BYTES, + }, + )?; + let mut bytes = Vec::new(); + file.take(read_bound) + .read_to_end(&mut bytes) + .map_err(|source| DocumentationError::RepositoryFileInspect { path, source })?; + if u64::try_from(bytes.len()).map_or(true, |length| length > MAX_REPOSITORY_FILE_BYTES) { + return Err(DocumentationError::RepositoryFileTooLarge { + path, + maximum: MAX_REPOSITORY_FILE_BYTES, + }); + } + let text = String::from_utf8(bytes) + .map_err(|source| DocumentationError::RepositoryFileEncoding { path, source })?; + Ok(RepositoryText { + identity, + path, + text, + }) +} + +fn open_error(path: &'static str, error: OpenRepositoryFileError) -> DocumentationError { + match error { + OpenRepositoryFileError::Io(source) => { + DocumentationError::RepositoryFileInspect { path, source } + } + OpenRepositoryFileError::NonRegular => DocumentationError::RepositoryFileNonRegular(path), + } +} + +#[cfg(test)] +#[path = "repository_text/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/repository_text/tests.rs b/xtask/src/documentation_integrity/repository_text/tests.rs new file mode 100644 index 0000000..b17d99a --- /dev/null +++ b/xtask/src/documentation_integrity/repository_text/tests.rs @@ -0,0 +1,76 @@ +use std::fs; +use std::io::Write; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::{MAX_REPOSITORY_FILE_BYTES, read}; + +#[test] +fn repository_policy_reads_are_utf8_and_bounded() -> Result<(), Box> { + let directory = TestDirectory::create("repository-text")?; + fs::write(directory.path().join("policy.txt"), "policy\n")?; + let root = RepositoryRoot::open(directory.path())?; + + assert_eq!(read(&root, "policy.txt")?.as_str(), "policy\n"); + + fs::write(directory.path().join("policy.txt"), [0xff])?; + assert!(matches!( + read(&root, "policy.txt"), + Err(super::DocumentationError::RepositoryFileEncoding { + path: "policy.txt", + .. + }) + )); + drop(root); + directory.close()?; + Ok(()) +} + +#[test] +fn repository_policy_identity_is_revalidated_after_admission() +-> Result<(), Box> { + let directory = TestDirectory::create("repository-text-identity")?; + let path = directory.path().join("policy.txt"); + let retained = directory.path().join("retained.txt"); + fs::write(&path, "admitted\n")?; + let root = RepositoryRoot::open(directory.path())?; + let policy = read(&root, "policy.txt")?; + + fs::rename(&path, &retained)?; + fs::write(&path, "replacement\n")?; + + assert!(matches!( + policy.verify(&root), + Err(super::DocumentationError::RepositoryFileChanged( + "policy.txt" + )) + )); + drop(root); + directory.close()?; + Ok(()) +} + +#[test] +fn repository_policy_reads_refuse_bytes_beyond_the_bound() -> Result<(), Box> +{ + let directory = TestDirectory::create("repository-text-bound")?; + let path = directory.path().join("policy.txt"); + let mut file = fs::File::create(&path)?; + let maximum = usize::try_from(MAX_REPOSITORY_FILE_BYTES)?; + file.write_all(&vec![b'x'; maximum])?; + file.write_all(b"x")?; + drop(file); + let root = RepositoryRoot::open(directory.path())?; + + assert!(matches!( + read(&root, "policy.txt"), + Err(super::DocumentationError::RepositoryFileTooLarge { + path: "policy.txt", + maximum: MAX_REPOSITORY_FILE_BYTES, + }) + )); + drop(root); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/tests.rs b/xtask/src/documentation_integrity/tests.rs new file mode 100644 index 0000000..2019568 --- /dev/null +++ b/xtask/src/documentation_integrity/tests.rs @@ -0,0 +1,49 @@ +//! This module owns documentation-integrity orchestration regression evidence. + +use std::error::Error; +use std::fs; +use std::io; +use std::path::Path; + +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +use super::{DocumentationError, check_with}; + +const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; + +#[test] +fn fixed_policy_replacement_during_tool_execution_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-policy-replacement")?; + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or_else(|| io::Error::other("workspace root is unavailable"))?; + let source = source + .to_str() + .ok_or_else(|| io::Error::other("workspace root is not UTF-8"))?; + run_git( + directory.path(), + &["clone", "--quiet", "--no-hardlinks", source, "repository"], + )?; + let repository = directory.path().join("repository"); + let result = check_with(&repository, |_, _, _, _| replace_lock(&repository)); + assert!(matches!( + result, + Err(DocumentationError::RepositoryFileChanged(LOCK_PATH)) + )); + Ok(()) +} + +fn replace_lock(repository: &Path) -> Result<(), DocumentationError> { + let lock = repository.join(LOCK_PATH); + let replacement = repository.join("replacement-package-lock.json"); + fs::write(&replacement, "{}\n").map_err(inspect)?; + fs::rename(replacement, lock).map_err(inspect) +} + +fn inspect(source: io::Error) -> DocumentationError { + DocumentationError::RepositoryFileInspect { + path: LOCK_PATH, + source, + } +} diff --git a/xtask/src/documentation_integrity/tool.rs b/xtask/src/documentation_integrity/tool.rs new file mode 100644 index 0000000..9d40efb --- /dev/null +++ b/xtask/src/documentation_integrity/tool.rs @@ -0,0 +1,76 @@ +//! This module owns admitted documentation-tool versions and arguments. + +use super::error::DocumentationError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum DocumentationTool { + Actionlint, + Lychee, + Markdownlint, +} + +impl DocumentationTool { + pub(super) const fn program(self) -> &'static str { + match self { + Self::Actionlint => "actionlint", + Self::Lychee => "lychee", + Self::Markdownlint => "markdownlint-cli2", + } + } + + pub(super) const fn install_version(self) -> &'static str { + match self { + Self::Actionlint => "1.7.12", + Self::Lychee => "0.21.0", + Self::Markdownlint => "0.23.2", + } + } + + pub(super) const fn expected_version(self) -> &'static str { + match self { + Self::Actionlint => "1.7.12", + Self::Lychee => "lychee 0.21.0", + Self::Markdownlint => "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + } + } + + pub(super) const fn version_arguments(self) -> &'static [&'static str] { + match self { + Self::Actionlint => &["-version"], + Self::Lychee => &["--version"], + Self::Markdownlint => &["--no-globs", "--version"], + } + } + + pub(super) const fn check_prefix(self) -> &'static [&'static str] { + match self { + Self::Actionlint => &["-shellcheck=", "-pyflakes="], + Self::Lychee => &[ + "--offline", + "--include-fragments", + "--no-progress", + "--format", + "detailed", + "--", + ], + Self::Markdownlint => &["--no-globs", "--"], + } + } + + pub(super) fn admit_version(self, observed: &str) -> Result<(), DocumentationError> { + let expected = self.expected_version(); + if observed == expected { + Ok(()) + } else { + Err(DocumentationError::VersionMismatch { + program: self.program(), + expected, + observed: observed.to_owned(), + }) + } + } +} + +#[cfg(test)] +#[path = "tool/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/tool/tests.rs b/xtask/src/documentation_integrity/tool/tests.rs new file mode 100644 index 0000000..7e8a739 --- /dev/null +++ b/xtask/src/documentation_integrity/tool/tests.rs @@ -0,0 +1,63 @@ +//! This module owns documentation-tool policy regression evidence. + +use super::DocumentationTool; +use crate::documentation_integrity::error::DocumentationError; + +#[test] +fn every_unreviewed_tool_version_is_refused() { + for tool in tools() { + assert!(matches!( + tool.admit_version("999.0.0"), + Err(DocumentationError::VersionMismatch { + program, + expected, + ref observed, + }) if program == tool.program() + && expected == tool.expected_version() + && observed == "999.0.0" + )); + } +} + +#[test] +fn every_reviewed_tool_version_is_admitted_exactly() { + for tool in tools() { + assert!(tool.admit_version(tool.expected_version()).is_ok()); + assert!(!tool.install_version().is_empty()); + } +} + +#[test] +fn tool_arguments_preserve_the_reviewed_execution_boundary() { + assert_eq!( + DocumentationTool::Markdownlint.version_arguments(), + ["--no-globs", "--version"] + ); + assert_eq!( + DocumentationTool::Markdownlint.check_prefix(), + ["--no-globs", "--"] + ); + assert_eq!( + DocumentationTool::Lychee.check_prefix(), + [ + "--offline", + "--include-fragments", + "--no-progress", + "--format", + "detailed", + "--", + ] + ); + assert_eq!( + DocumentationTool::Actionlint.check_prefix(), + ["-shellcheck=", "-pyflakes="] + ); +} + +const fn tools() -> [DocumentationTool; 3] { + [ + DocumentationTool::Markdownlint, + DocumentationTool::Lychee, + DocumentationTool::Actionlint, + ] +} diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs new file mode 100644 index 0000000..7a59740 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -0,0 +1,282 @@ +//! This module owns the CI documentation-job execution contract. + +mod reviewed_step; + +use yaml_rust2::{Yaml, YamlLoader}; + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text::{self, RepositoryText}; +use reviewed_step::{DocumentationStep, REVIEWED_STEPS, steps_have_reviewed_membership}; + +const CI_PATH: &str = ".github/workflows/ci.yml"; +const CHECKOUT_ACTION: &str = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; +const CHECKOUT_ACTION_PREFIX: &str = "actions/checkout@"; +const DOCUMENTATION_JOB_FIELDS: &[&str] = &["name", "runs-on", "timeout-minutes", "steps"]; +const DOCUMENTATION_JOB_NAME: &str = "Documentation and workflow integrity"; +const DOCUMENTATION_RUNNER: &str = "ubuntu-latest"; +const DOCUMENTATION_TIMEOUT_MINUTES: i64 = 10; +const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; +const SETUP_NODE_ACTION_PREFIX: &str = "actions/setup-node@"; +const NODE_VERSION: &str = "24.18.0"; + +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result { + let workflow = repository_text::read(repository_root, CI_PATH)?; + admit(workflow.as_str())?; + workflow.verify(repository_root)?; + Ok(workflow) +} + +fn admit(workflow: &str) -> Result<(), DocumentationError> { + let steps = admitted_steps(workflow)?; + if steps.as_slice() == REVIEWED_STEPS { + return Ok(()); + } + if steps_have_reviewed_membership(&steps) { + Err(contract( + "documentation job steps execute in reviewed order", + )) + } else { + Err(contract(concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ))) + } +} + +fn admitted_steps(workflow: &str) -> Result, DocumentationError> { + let documents = YamlLoader::load_from_str(workflow).map_err(|source| { + DocumentationError::RepositoryYaml { + path: CI_PATH, + source, + } + })?; + let [document] = documents.as_slice() else { + return Err(contract("workflow contains exactly one YAML document")); + }; + if !triggers_are_reviewed(document) { + return Err(contract( + "workflow runs on reviewed push and pull request triggers", + )); + } + if !permissions_are_read_only(document) { + return Err(contract("workflow permissions are read-only")); + } + reviewed_steps(documentation_steps(document)?) +} + +fn triggers_are_reviewed(document: &Yaml) -> bool { + let triggers = &document["on"]; + let push = &triggers["push"]; + let branches = &push["branches"]; + mapping_has_exact_fields(triggers, &["push", "pull_request"]) + && mapping_has_exact_fields(push, &["branches"]) + && matches!(branches.as_vec().map(Vec::as_slice), Some([branch]) if branch.as_str() == Some("main")) + && triggers["pull_request"].is_null() +} + +fn permissions_are_read_only(document: &Yaml) -> bool { + let permissions = &document["permissions"]; + mapping_has_exact_fields(permissions, &["contents"]) + && permissions["contents"].as_str() == Some("read") +} + +fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError> { + if !document["defaults"].is_badvalue() || !document["env"].is_badvalue() { + return Err(contract( + "workflow does not override documentation execution", + )); + } + let job = &document["jobs"]["documentation"]; + if !job["if"].is_badvalue() { + return Err(contract("documentation job is unguarded")); + } + if !job["continue-on-error"].is_badvalue() { + return Err(contract("documentation job is failure-intolerant")); + } + if job["name"].as_str() != Some(DOCUMENTATION_JOB_NAME) { + return Err(contract("documentation job name is reviewed")); + } + if job["runs-on"].as_str() != Some(DOCUMENTATION_RUNNER) { + return Err(contract("documentation job uses ubuntu-latest")); + } + if job["timeout-minutes"].as_i64() != Some(DOCUMENTATION_TIMEOUT_MINUTES) { + return Err(contract("documentation job timeout is ten minutes")); + } + if !mapping_has_exact_fields(job, DOCUMENTATION_JOB_FIELDS) { + return Err(contract("documentation job fields are reviewed")); + } + let Some(steps) = job["steps"].as_vec() else { + return Err(contract("workflow defines documentation job steps")); + }; + Ok(steps) +} + +fn reviewed_steps(steps: &[Yaml]) -> Result, DocumentationError> { + let mut admitted = Vec::new(); + let mut actions = Vec::new(); + for step in steps { + if let Some(action) = admit_action(step)? { + actions.push(action); + admitted.push(match action { + DocumentationAction::Checkout => DocumentationStep::Checkout, + DocumentationAction::Node => DocumentationStep::Node, + }); + } else if let Some(run) = admit_run(step)? { + admitted.push(run); + } else { + return Err(contract("documentation job steps are reviewed")); + } + } + if actions + .iter() + .filter(|action| **action == DocumentationAction::Node) + .count() + != 1 + { + return Err(contract( + "documentation job installs pinned Node.js exactly once", + )); + } + if actions.as_slice() != REVIEWED_ACTIONS { + return Err(contract( + "documentation job actions execute in reviewed order", + )); + } + Ok(admitted) +} + +fn admit_action(step: &Yaml) -> Result, DocumentationError> { + let uses = &step["uses"]; + if uses.is_badvalue() { + return Ok(None); + } + let Some(action) = uses.as_str() else { + return Err(contract("documentation job action values are strings")); + }; + if action.starts_with(CHECKOUT_ACTION_PREFIX) { + return admit_checkout(step, action).map(Some); + } + if action.starts_with(SETUP_NODE_ACTION_PREFIX) { + return admit_node_setup(step, action).map(Some); + } + Err(contract("documentation job action steps are reviewed")) +} + +fn admit_checkout(step: &Yaml, action: &str) -> Result { + if action != CHECKOUT_ACTION { + return Err(contract("documentation checkout action is pinned")); + } + admit_action_execution( + step, + "documentation checkout is unguarded", + "documentation checkout is failure-intolerant", + )?; + let exact = step["with"] + .as_hash() + .is_some_and(|configuration| configuration.len() == 1) + && step["with"]["persist-credentials"].as_bool() == Some(false); + if !exact { + return Err(contract("documentation checkout configuration is exact")); + } + Ok(DocumentationAction::Checkout) +} + +fn admit_node_setup(step: &Yaml, action: &str) -> Result { + if action != SETUP_NODE_ACTION { + return Err(contract("documentation Node.js setup action is pinned")); + } + admit_action_execution( + step, + "documentation Node.js setup is unguarded", + "documentation Node.js setup is failure-intolerant", + )?; + let exact = step["with"] + .as_hash() + .is_some_and(|configuration| configuration.len() == 1) + && step["with"]["node-version"].as_str() == Some(NODE_VERSION); + if !exact { + return Err(contract("documentation Node.js version is 24.18.0")); + } + Ok(DocumentationAction::Node) +} + +fn admit_action_execution( + step: &Yaml, + guard_requirement: &'static str, + failure_requirement: &'static str, +) -> Result<(), DocumentationError> { + if !step["if"].is_badvalue() { + return Err(contract(guard_requirement)); + } + if !step["continue-on-error"].is_badvalue() { + return Err(contract(failure_requirement)); + } + if !step["run"].is_badvalue() { + return Err(contract("documentation action steps do not define run")); + } + if !mapping_has_exact_fields(step, &["name", "uses", "with"]) { + return Err(contract("documentation action step fields are reviewed")); + } + Ok(()) +} + +fn admit_run(step: &Yaml) -> Result, DocumentationError> { + let run = &step["run"]; + if run.is_badvalue() { + return Ok(None); + } + if !step["if"].is_badvalue() { + return Err(contract("documentation job run steps are unguarded")); + } + if !step["continue-on-error"].is_badvalue() { + return Err(contract( + "documentation job run steps are failure-intolerant", + )); + } + if !mapping_has_exact_fields(step, &["name", "run"]) { + return Err(contract("documentation job run step fields are reviewed")); + } + let Some(run) = run.as_str() else { + return Err(contract("documentation job run values are strings")); + }; + let Some(admitted) = DocumentationStep::from_run(run.trim_end_matches('\n')) else { + return Err(contract(concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ))); + }; + Ok(Some(admitted)) +} + +fn mapping_has_exact_fields(mapping: &Yaml, fields: &[&str]) -> bool { + mapping.as_hash().is_some_and(|mapping| { + mapping.len() == fields.len() + && mapping + .keys() + .all(|field| field.as_str().is_some_and(|field| fields.contains(&field))) + }) +} + +const REVIEWED_ACTIONS: &[DocumentationAction] = + &[DocumentationAction::Checkout, DocumentationAction::Node]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DocumentationAction { + Checkout, + Node, +} + +const fn contract(requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { + path: CI_PATH, + requirement, + } +} + +#[cfg(test)] +#[path = "workflow_contract/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/workflow_contract/action_steps.rs b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs new file mode 100644 index 0000000..1e75284 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs @@ -0,0 +1,106 @@ +//! This module owns documentation-job action-step policy regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +const CHECKOUT_STEP: &str = concat!( + " - name: Check out repository\n", + " uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1\n", + " with:\n", + " persist-credentials: false\n" +); +const NODE_STEP: &str = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" +); +const INSTALL_STEP: &str = concat!( + " - name: Install documentation tools\n", + " run: |\n", + " documentation_tools=\"$RUNNER_TEMP/documentation-tools\"\n", + " scripts/install_documentation_tools.sh \"$documentation_tools\"\n", + " printf '%s\\n' \\\n", + " \"$documentation_tools/bin\" \\\n", + " \"$documentation_tools/npm/node_modules/.bin\" >> \"$GITHUB_PATH\"\n" +); + +#[test] +fn checkout_of_an_unreviewed_revision_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " persist-credentials: false", + " persist-credentials: false\n ref: main", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation checkout configuration is exact", + }) + )); +} + +#[test] +fn drifted_checkout_action_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/checkout@0123456789abcdef0123456789abcdef01234567", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation checkout action is pinned", + }) + )); +} + +#[test] +fn unreviewed_action_steps_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Install Rust\n", + concat!( + " - name: Unreviewed action\n", + " uses: example/action@0123456789abcdef0123456789abcdef01234567\n", + " - name: Install Rust\n" + ), + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job action steps are reviewed", + }) + )); +} + +#[test] +fn documentation_actions_execute_in_reviewed_order() { + let workflow = WORKFLOW + .replace(CHECKOUT_STEP, "CHECKOUT_STEP_PLACEHOLDER\n") + .replace(NODE_STEP, CHECKOUT_STEP) + .replace("CHECKOUT_STEP_PLACEHOLDER\n", NODE_STEP); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job actions execute in reviewed order", + }) + )); +} + +#[test] +fn actions_and_commands_execute_in_one_reviewed_order() { + let moved = format!("{INSTALL_STEP}{NODE_STEP}"); + let workflow = WORKFLOW + .replace(NODE_STEP, "") + .replace(INSTALL_STEP, &moved); + + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job steps execute in reviewed order", + }) + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/execution_context.rs b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs new file mode 100644 index 0000000..0ede2bf --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs @@ -0,0 +1,85 @@ +//! This module owns documentation-job execution-context regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +#[test] +fn custom_run_shell_cannot_impersonate_a_required_command() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n shell: \"echo {0}\"\n run:", + ); + + assert_contract(&workflow, "documentation job run step fields are reviewed"); +} + +#[test] +fn workflow_run_defaults_cannot_replace_required_commands() { + let workflow = WORKFLOW.replace( + "jobs:\n", + "defaults:\n run:\n shell: \"echo {0}\"\njobs:\n", + ); + + assert_contract( + &workflow, + "workflow does not override documentation execution", + ); +} + +#[test] +fn workflow_cannot_grant_write_all_permissions() { + let workflow = WORKFLOW.replace("permissions:\n contents: read", "permissions: write-all"); + + assert_contract(&workflow, "workflow permissions are read-only"); +} + +#[test] +fn workflow_content_permission_cannot_grant_write_authority() { + let workflow = WORKFLOW.replace(" contents: read", " contents: write"); + + assert_contract(&workflow, "workflow permissions are read-only"); +} + +#[test] +fn job_run_defaults_cannot_replace_required_commands() { + let workflow = WORKFLOW.replace( + " steps:\n", + " defaults:\n run:\n shell: \"echo {0}\"\n steps:\n", + ); + + assert_contract(&workflow, "documentation job fields are reviewed"); +} + +#[test] +fn documentation_job_cannot_move_to_an_unreviewed_runner() { + let workflow = WORKFLOW.replace("runs-on: ubuntu-latest", "runs-on: self-hosted"); + + assert_contract(&workflow, "documentation job uses ubuntu-latest"); +} + +#[test] +fn documentation_job_cannot_extend_its_reviewed_deadline() { + let workflow = WORKFLOW.replace("timeout-minutes: 10", "timeout-minutes: 60"); + + assert_contract(&workflow, "documentation job timeout is ten minutes"); +} + +#[test] +fn action_environment_cannot_change_reviewed_execution() { + let workflow = WORKFLOW.replace( + " - name: Install pinned Node.js\n uses:", + " - name: Install pinned Node.js\n env:\n NODE_OPTIONS: --require=./hook.js\n uses:", + ); + + assert_contract(&workflow, "documentation action step fields are reviewed"); +} + +fn assert_contract(workflow: &str, requirement: &'static str) { + assert!(matches!( + admit(workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: observed, + }) if observed == requirement + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/node_setup.rs b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs new file mode 100644 index 0000000..f119476 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs @@ -0,0 +1,122 @@ +//! This module owns documentation-job Node.js setup policy regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +#[test] +fn documentation_job_requires_the_pinned_node_action_once() { + let setup = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" + ); + let missing = WORKFLOW.replace(setup, ""); + assert!(matches!( + admit(&missing), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); +} + +#[test] +fn duplicate_pinned_node_setup_is_refused_for_multiplicity() { + let setup = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" + ); + let workflow = WORKFLOW.replace(setup, &setup.repeat(2)); + + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); +} + +#[test] +fn drifted_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", + "actions/setup-node@0123456789abcdef0123456789abcdef01234567", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); +} + +#[test] +fn additional_unpinned_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Install documentation tools\n", + concat!( + " - name: Replace Node.js\n", + " uses: actions/setup-node@0123456789abcdef0123456789abcdef01234567\n", + " with:\n", + " node-version: 24.18.1\n", + " - name: Install documentation tools\n" + ), + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); +} + +#[test] +fn guarded_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " if: false\n uses: actions/setup-node@", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup is unguarded", + }) + )); +} + +#[test] +fn failure_tolerant_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " continue-on-error: true\n uses: actions/setup-node@", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup is failure-intolerant", + }) + )); +} + +#[test] +fn documentation_job_requires_the_reviewed_node_version() { + let workflow = WORKFLOW.replace( + " node-version: 24.18.0", + " node-version: 24.18.1", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js version is 24.18.0", + }) + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs new file mode 100644 index 0000000..0023d62 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs @@ -0,0 +1,61 @@ +//! This module owns the exact documentation-job step sequence. + +const MALFORMED_INPUT_COMMAND: &str = "cargo xtask documentation-refusal-check"; +const INSTALL_TOOLS_COMMAND: &str = r#"documentation_tools="$RUNNER_TEMP/documentation-tools" +scripts/install_documentation_tools.sh "$documentation_tools" +printf '%s\n' \ + "$documentation_tools/bin" \ + "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH""#; +const DIFF_CHECK_COMMAND: &str = r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#; + +/// The only admitted documentation-job execution sequence. +pub(super) const REVIEWED_STEPS: &[DocumentationStep] = &[ + DocumentationStep::Checkout, + DocumentationStep::Rustup, + DocumentationStep::Node, + DocumentationStep::InstallTools, + DocumentationStep::MalformedInputs, + DocumentationStep::Verify, + DocumentationStep::DiffCheck, +]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// One semantically reviewed documentation-job step. +pub(super) enum DocumentationStep { + /// Opens the exact repository revision without retained credentials. + Checkout, + /// Confirms the pinned Rust toolchain selected by repository policy. + Rustup, + /// Installs the pinned Node.js runtime. + Node, + /// Installs the lockfile-bound documentation tools. + InstallTools, + /// Runs malformed-input regression evidence. + MalformedInputs, + /// Runs the Rust documentation-integrity boundary. + Verify, + /// Refuses whitespace errors across the reviewed tree. + DiffCheck, +} + +impl DocumentationStep { + /// Classifies one exact reviewed `run` body. + pub(super) fn from_run(run: &str) -> Option { + match run { + "rustup show" => Some(Self::Rustup), + INSTALL_TOOLS_COMMAND => Some(Self::InstallTools), + MALFORMED_INPUT_COMMAND => Some(Self::MalformedInputs), + "cargo xtask documentation-integrity-check" => Some(Self::Verify), + DIFF_CHECK_COMMAND => Some(Self::DiffCheck), + _ => None, + } + } +} + +/// Reports whether every reviewed step occurs exactly once. +pub(super) fn steps_have_reviewed_membership(steps: &[DocumentationStep]) -> bool { + steps.len() == REVIEWED_STEPS.len() + && REVIEWED_STEPS + .iter() + .all(|required| steps.iter().filter(|step| *step == required).count() == 1) +} diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs new file mode 100644 index 0000000..b4e3629 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -0,0 +1,257 @@ +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +mod action_steps; +mod execution_context; +mod node_setup; + +const WORKFLOW: &str = r#"name: CI +on: + push: + branches: [main] + pull_request: +permissions: + contents: read +jobs: + documentation: + name: Documentation and workflow integrity + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Install Rust + run: rustup show + - name: Install pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.18.0 + - name: Install documentation tools + run: | + documentation_tools="$RUNNER_TEMP/documentation-tools" + scripts/install_documentation_tools.sh "$documentation_tools" + printf '%s\n' \ + "$documentation_tools/bin" \ + "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" + - name: Verify malformed inputs + run: cargo xtask documentation-refusal-check + - name: Verify + run: cargo xtask documentation-integrity-check + - name: Check whitespace + run: git diff --check "$(git hash-object -t tree /dev/null)" HEAD + next-job: + steps: [] +"#; + +#[test] +fn documentation_job_delegates_once_to_the_rust_boundary() { + assert!(super::admit(WORKFLOW).is_ok()); +} + +#[test] +fn documentation_job_requires_reviewed_workflow_triggers() { + for workflow in [ + WORKFLOW.replace(" pull_request:\n", ""), + WORKFLOW.replacen( + "on:\n push:\n branches: [main]\n pull_request:\n", + "on:\n workflow_dispatch:\n", + 1, + ), + WORKFLOW.replace( + " pull_request:\n", + " pull_request:\n paths:\n - README.md\n", + ), + ] { + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "workflow runs on reviewed push and pull request triggers", + }) + )); + } +} + +#[test] +fn documentation_job_requires_the_malformed_input_regressions() { + let steps = super::REVIEWED_STEPS + .iter() + .copied() + .filter(|step| *step != super::DocumentationStep::MalformedInputs) + .collect::>(); + + assert!(!super::steps_have_reviewed_membership(&steps)); +} + +#[test] +fn documentation_job_refuses_unclassified_step_mappings() { + let workflow = WORKFLOW.replace( + " - name: Verify malformed inputs\n", + " - name: Unreviewed placeholder\n - name: Verify malformed inputs\n", + ); + + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job steps are reviewed", + }) + )); +} + +#[test] +fn documentation_job_refuses_python_execution() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Legacy checker\n run: python3 scripts/check_markdown.py\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ), + }) + )); +} + +#[test] +fn inert_yaml_cannot_impersonate_documentation_commands() { + let workflow = r#"name: CI +on: + push: + branches: [main] + pull_request: +permissions: + contents: read +jobs: + documentation: + name: Documentation and workflow integrity + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Install pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.18.0 + # run: rustup show + - name: "run: cargo xtask documentation-integrity-check" +"#; + assert!(matches!( + super::admit(workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job steps are reviewed", + }) + )); +} + +#[test] +fn non_string_run_values_are_refused() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Invalid executable\n run: true\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run values are strings", + }) + )); +} + +#[test] +fn guarded_required_commands_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n if: false\n run:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run steps are unguarded", + }) + )); +} + +#[test] +fn guarded_documentation_jobs_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace(" documentation:\n", " documentation:\n if: false\n"); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job is unguarded", + }) + )); +} + +#[test] +fn failure_tolerant_documentation_jobs_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " documentation:\n", + " documentation:\n continue-on-error: true\n", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job is failure-intolerant", + }) + )); +} + +#[test] +fn failure_tolerant_required_commands_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n continue-on-error: true\n run:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run steps are failure-intolerant", + }) + )); +} + +#[test] +fn unreviewed_python_executables_are_refused() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Unreviewed executable\n run: python --version\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ), + }) + )); +} + +#[test] +fn committed_documentation_job_uses_the_rust_boundary() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + super::check(&repository_root)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} diff --git a/xtask/src/fuzz_campaign.rs b/xtask/src/fuzz_campaign.rs index 87c7e38..9e0361d 100644 --- a/xtask/src/fuzz_campaign.rs +++ b/xtask/src/fuzz_campaign.rs @@ -5,7 +5,6 @@ mod corpus; mod error; mod execution; mod policy; -mod process; mod profile; mod target; #[cfg(test)] @@ -112,7 +111,7 @@ fn execute_plans( let plans = target::registered(repository_root, policy)? .into_iter() .map(|target| CommandPlan::new(policy, operation, target)) - .collect::>(); + .collect::, _>>()?; execution::run(repository_root, name, &plans)?; Ok(()) } diff --git a/xtask/src/fuzz_campaign/command.rs b/xtask/src/fuzz_campaign/command.rs index 1771416..4deea8c 100644 --- a/xtask/src/fuzz_campaign/command.rs +++ b/xtask/src/fuzz_campaign/command.rs @@ -6,7 +6,7 @@ mod tests; use std::ffi::OsString; use std::time::Duration; -use super::policy::CampaignPolicy; +use super::policy::{CampaignPolicy, PolicyError}; use super::profile::CampaignProfile; use super::target::FuzzTarget; @@ -38,7 +38,7 @@ impl CommandPlan { policy: &CampaignPolicy, operation: CampaignOperation, target: FuzzTarget, - ) -> Self { + ) -> Result { let mut arguments = vec![ OsString::from(format!("+{}", policy.toolchain())), OsString::from("fuzz"), @@ -46,7 +46,7 @@ impl CommandPlan { let (deadline, output_mode, refused_output_marker) = match operation { CampaignOperation::Build => { arguments.extend([OsString::from("build"), OsString::from(target.as_str())]); - (None, OutputMode::Inherit, None) + (Some(policy.build_timeout()), OutputMode::Inherit, None) } CampaignOperation::Minimize => { arguments.extend([OsString::from("cmin"), OsString::from(target.as_str())]); @@ -60,16 +60,20 @@ impl CommandPlan { CampaignOperation::Run(profile) => { arguments.extend([OsString::from("run"), OsString::from(target.as_str())]); push_fuzzer_arguments(&mut arguments, policy.seconds_per_target(profile), policy); - (None, OutputMode::Inherit, None) + ( + Some(policy.run_timeout(profile)?), + OutputMode::Inherit, + None, + ) } }; - Self { + Ok(Self { target, arguments, deadline, output_mode, refused_output_marker, - } + }) } pub(super) const fn target(&self) -> &FuzzTarget { diff --git a/xtask/src/fuzz_campaign/command/tests.rs b/xtask/src/fuzz_campaign/command/tests.rs index eccf143..aab93c0 100644 --- a/xtask/src/fuzz_campaign/command/tests.rs +++ b/xtask/src/fuzz_campaign/command/tests.rs @@ -15,7 +15,7 @@ fn run_plan_preserves_every_reviewed_resource_bound() -> Result<(), Box Result<(), Box Result<(), Box> { + let policy = policy()?; + let target = FuzzTarget::admit("segment_format".to_owned())?; + let plan = CommandPlan::new(&policy, CampaignOperation::Build, target)?; + + assert_eq!(plan.deadline(), Some(Duration::from_mins(10))); assert_eq!(plan.output_mode(), OutputMode::Inherit); Ok(()) } @@ -40,7 +51,7 @@ fn run_plan_preserves_every_reviewed_resource_bound() -> Result<(), Box Result<(), Box> { let policy = policy()?; let target = FuzzTarget::admit("blob_hasher".to_owned())?; - let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target); + let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target)?; assert_eq!(plan.deadline(), Some(Duration::from_mins(2))); assert_eq!( plan.refused_output_marker(), diff --git a/xtask/src/fuzz_campaign/execution.rs b/xtask/src/fuzz_campaign/execution.rs index fa0bd69..fc1fd92 100644 --- a/xtask/src/fuzz_campaign/execution.rs +++ b/xtask/src/fuzz_campaign/execution.rs @@ -12,7 +12,9 @@ pub(crate) use error::ExecutionError; use error::{TargetFailure, TargetFailureReason}; use super::command::{CommandPlan, OutputMode}; -use super::process::{self, ProcessError, ProcessOutput}; +use crate::bounded_process::{self, ProcessError, ProcessOutput}; + +const CARGO_FUZZ_PROCESS: &str = "cargo-fuzz"; trait CommandRunner { fn execute( @@ -34,11 +36,14 @@ impl CommandRunner for SystemRunner { command.args(plan.arguments()).current_dir(repository_root); match plan.output_mode() { OutputMode::Capture => { - let output = process::capture(&mut command, plan.deadline())?; + let output = + bounded_process::capture(CARGO_FUZZ_PROCESS, &mut command, plan.deadline())?; replay(&output)?; Ok(output) } - OutputMode::Inherit => process::status(&mut command), + OutputMode::Inherit => { + bounded_process::status(CARGO_FUZZ_PROCESS, &mut command, plan.deadline()) + } } } } @@ -72,7 +77,11 @@ fn write_output( writer .write_all(bytes) .and_then(|()| writer.flush()) - .map_err(|source| ProcessError::Io { action, source }) + .map_err(|source| ProcessError::Io { + program: CARGO_FUZZ_PROCESS, + action, + source, + }) } fn execute_all( diff --git a/xtask/src/fuzz_campaign/execution/error.rs b/xtask/src/fuzz_campaign/execution/error.rs index 3ccad36..2311fdf 100644 --- a/xtask/src/fuzz_campaign/execution/error.rs +++ b/xtask/src/fuzz_campaign/execution/error.rs @@ -3,7 +3,7 @@ use std::error::Error; use std::fmt; -use crate::fuzz_campaign::process::ProcessError; +use crate::bounded_process::ProcessError; pub(crate) struct ExecutionError { pub(super) operation: &'static str, diff --git a/xtask/src/fuzz_campaign/execution/tests.rs b/xtask/src/fuzz_campaign/execution/tests.rs index 34ce815..0497167 100644 --- a/xtask/src/fuzz_campaign/execution/tests.rs +++ b/xtask/src/fuzz_campaign/execution/tests.rs @@ -4,9 +4,9 @@ use std::io; use std::path::Path; use super::{CommandRunner, execute_all}; +use crate::bounded_process::{ProcessError, ProcessOutput}; use crate::fuzz_campaign::command::{CampaignOperation, CommandPlan}; use crate::fuzz_campaign::policy::CampaignPolicy; -use crate::fuzz_campaign::process::{ProcessError, ProcessOutput}; use crate::fuzz_campaign::profile::CampaignProfile; use crate::fuzz_campaign::target::FuzzTarget; @@ -35,7 +35,7 @@ fn every_target_runs_after_an_earlier_failure() -> Result<(), Box> { fn swallowed_minimization_failure_is_refused() -> Result<(), Box> { let policy = policy()?; let target = FuzzTarget::admit("first".to_owned())?; - let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target); + let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target)?; let mut runner = ScriptedRunner::new([output(true, b"Failed to minimize corpus: signal 6", b"")]); let Err(error) = execute_all(Path::new("."), "corpus minimization", &[plan], &mut runner) @@ -69,7 +69,7 @@ fn plans(operation: CampaignOperation) -> Result, Box Result> { fn output(succeeded: bool, stdout: &[u8], stderr: &[u8]) -> ProcessOutput { ProcessOutput { + code: Some(i32::from(!succeeded)), succeeded, stdout: stdout.to_vec(), stderr: stderr.to_vec(), @@ -103,6 +104,7 @@ impl CommandRunner for RefusingRunner { _plan: &CommandPlan, ) -> Result { Err(ProcessError::Io { + program: "cargo-fuzz", action: "spawn", source: io::Error::other("scripted refusal"), }) @@ -126,7 +128,10 @@ impl CommandRunner for ScriptedRunner { ) -> Result { self.observed.push(plan.target().as_str().to_owned()); let Some(output) = self.outcomes.pop_front() else { - return Err(ProcessError::MissingStream("scripted outcome")); + return Err(ProcessError::MissingStream { + program: "cargo-fuzz", + stream: "scripted outcome", + }); }; Ok(output) } diff --git a/xtask/src/fuzz_campaign/policy.rs b/xtask/src/fuzz_campaign/policy.rs index d92c85c..03988d9 100644 --- a/xtask/src/fuzz_campaign/policy.rs +++ b/xtask/src/fuzz_campaign/policy.rs @@ -8,6 +8,7 @@ mod tests; use std::collections::BTreeMap; use std::fs; use std::path::Path; +use std::time::Duration; use super::profile::CampaignProfile; pub(crate) use error::PolicyError; @@ -16,8 +17,10 @@ use syntax::{bounded, is_dated_nightly, is_exact_version, parse_assignments, val pub(super) struct CampaignPolicy { cargo_fuzz_version: String, toolchain: String, + build_timeout_seconds: u64, input_timeout_seconds: u64, max_input_bytes: u64, + process_grace_seconds: u64, rss_limit_mb: u64, smoke_seconds_per_target: u64, scheduled_seconds_per_target: u64, @@ -54,6 +57,10 @@ impl CampaignPolicy { pub(super) fn environment(&self, profile: CampaignProfile) -> BTreeMap<&'static str, String> { [ ("CARGO_FUZZ_VERSION", self.cargo_fuzz_version.clone()), + ( + "FUZZ_BUILD_TIMEOUT_SECONDS", + self.build_timeout_seconds.to_string(), + ), ( "FUZZ_CMIN_SECONDS_PER_TARGET", self.cmin_seconds_per_target.to_string(), @@ -69,6 +76,10 @@ impl CampaignPolicy { self.input_timeout_seconds.to_string(), ), ("FUZZ_MAX_INPUT_BYTES", self.max_input_bytes.to_string()), + ( + "FUZZ_PROCESS_GRACE_SECONDS", + self.process_grace_seconds.to_string(), + ), ("FUZZ_RSS_LIMIT_MB", self.rss_limit_mb.to_string()), ( "FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS", @@ -96,6 +107,17 @@ impl CampaignPolicy { self.cmin_seconds_per_target } + pub(super) const fn build_timeout(&self) -> Duration { + Duration::from_secs(self.build_timeout_seconds) + } + + pub(super) fn run_timeout(&self, profile: CampaignProfile) -> Result { + self.seconds_per_target(profile) + .checked_add(self.process_grace_seconds) + .map(Duration::from_secs) + .ok_or(PolicyError::CampaignDeadline) + } + pub(super) const fn corpus_max_bytes(&self) -> u64 { self.corpus_max_bytes } @@ -124,8 +146,10 @@ impl CampaignPolicy { Ok(Self { cargo_fuzz_version, toolchain, + build_timeout_seconds: bounded(values, "FUZZ_BUILD_TIMEOUT_SECONDS", 60, 3_600)?, input_timeout_seconds: bounded(values, "FUZZ_INPUT_TIMEOUT_SECONDS", 1, 60)?, max_input_bytes: bounded(values, "FUZZ_MAX_INPUT_BYTES", 1, 1_048_576)?, + process_grace_seconds: bounded(values, "FUZZ_PROCESS_GRACE_SECONDS", 1, 600)?, rss_limit_mb: bounded(values, "FUZZ_RSS_LIMIT_MB", 128, 8_192)?, smoke_seconds_per_target: bounded(values, "FUZZ_SMOKE_SECONDS_PER_TARGET", 1, 60)?, scheduled_seconds_per_target: bounded( diff --git a/xtask/src/fuzz_campaign/policy/error.rs b/xtask/src/fuzz_campaign/policy/error.rs index f69af72..c08e745 100644 --- a/xtask/src/fuzz_campaign/policy/error.rs +++ b/xtask/src/fuzz_campaign/policy/error.rs @@ -14,6 +14,7 @@ pub(crate) enum PolicyError { maximum: u64, }, CampaignOrder, + CampaignDeadline, CorpusCapacity, InvalidInteger(&'static str), InvalidToolchain, @@ -51,6 +52,9 @@ impl fmt::Display for PolicyError { Self::CampaignOrder => { formatter.write_str("scheduled fuzzing must exceed the smoke budget") } + Self::CampaignDeadline => { + formatter.write_str("fuzz process deadline exceeds the duration range") + } Self::CorpusCapacity => { formatter.write_str("corpus bytes cannot be smaller than one input") } @@ -85,6 +89,7 @@ impl Error for PolicyError { match self { Self::Read { source, .. } => Some(source), Self::Bound { .. } + | Self::CampaignDeadline | Self::CampaignOrder | Self::CorpusCapacity | Self::InvalidInteger(_) diff --git a/xtask/src/fuzz_campaign/policy/syntax.rs b/xtask/src/fuzz_campaign/policy/syntax.rs index 7a5162c..f021a8f 100644 --- a/xtask/src/fuzz_campaign/policy/syntax.rs +++ b/xtask/src/fuzz_campaign/policy/syntax.rs @@ -4,14 +4,16 @@ use std::collections::BTreeMap; use super::PolicyError; -const EXPECTED_KEYS: [&str; 13] = [ +const EXPECTED_KEYS: [&str; 15] = [ "CARGO_FUZZ_VERSION", + "FUZZ_BUILD_TIMEOUT_SECONDS", "FUZZ_CMIN_SECONDS_PER_TARGET", "FUZZ_CORPUS_MAX_BYTES", "FUZZ_CORPUS_MAX_FILES", "FUZZ_CORPUS_RETENTION_DAYS", "FUZZ_INPUT_TIMEOUT_SECONDS", "FUZZ_MAX_INPUT_BYTES", + "FUZZ_PROCESS_GRACE_SECONDS", "FUZZ_RSS_LIMIT_MB", "FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS", "FUZZ_SCHEDULED_SECONDS_PER_TARGET", diff --git a/xtask/src/fuzz_campaign/policy/tests.rs b/xtask/src/fuzz_campaign/policy/tests.rs index 4d2cbfb..6da00d0 100644 --- a/xtask/src/fuzz_campaign/policy/tests.rs +++ b/xtask/src/fuzz_campaign/policy/tests.rs @@ -9,9 +9,11 @@ fn repository_policy_preserves_every_reviewed_runtime_bound() -> Result<(), Poli let policy = CampaignPolicy::parse(POLICY)?; assert_eq!(policy.cargo_fuzz_version, "0.13.2"); assert_eq!(policy.toolchain, "nightly-2026-07-24"); + assert_eq!(policy.build_timeout_seconds, 600); assert_eq!(policy.smoke_seconds_per_target, 15); assert_eq!(policy.scheduled_seconds_per_target, 600); assert_eq!(policy.cmin_seconds_per_target, 120); + assert_eq!(policy.process_grace_seconds, 60); assert_eq!(policy.input_timeout_seconds, 5); assert_eq!(policy.max_input_bytes, 1_048_576); assert_eq!(policy.rss_limit_mb, 1_024); diff --git a/xtask/src/fuzz_campaign/process.rs b/xtask/src/fuzz_campaign/process.rs deleted file mode 100644 index aebfd32..0000000 --- a/xtask/src/fuzz_campaign/process.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! This module owns bounded cargo-fuzz child-process collection. - -mod error; - -use std::io; -use std::process::{Child, Command, ExitStatus, Stdio}; -use std::thread::{self, JoinHandle}; -use std::time::{Duration, Instant}; - -pub(super) use error::ProcessError; - -use crate::process_output::{BoundedBytes, bounded_bytes}; - -const OUTPUT_LIMIT: usize = 1_048_576; - -pub(super) struct ProcessOutput { - pub(super) succeeded: bool, - pub(super) stdout: Vec, - pub(super) stderr: Vec, -} - -pub(super) fn status(command: &mut Command) -> Result { - let status = command.status().map_err(|source| ProcessError::Io { - action: "wait for", - source, - })?; - Ok(ProcessOutput { - succeeded: status.success(), - stdout: Vec::new(), - stderr: Vec::new(), - }) -} - -pub(super) fn capture( - command: &mut Command, - deadline: Option, -) -> Result { - command.stdout(Stdio::piped()).stderr(Stdio::piped()); - let mut child = command.spawn().map_err(|source| ProcessError::Io { - action: "spawn", - source, - })?; - let stdout = child - .stdout - .take() - .ok_or(ProcessError::MissingStream("stdout")); - let stderr = child - .stderr - .take() - .ok_or(ProcessError::MissingStream("stderr")); - let (stdout, stderr) = match (stdout, stderr) { - (Ok(stdout), Ok(stderr)) => (stdout, stderr), - (Err(error), _) | (_, Err(error)) => return Err(cleanup(&mut child, error)), - }; - let stdout_reader = match start_reader("stdout", stdout) { - Ok(reader) => reader, - Err(error) => return Err(cleanup(&mut child, error)), - }; - let stderr_reader = match start_reader("stderr", stderr) { - Ok(reader) => reader, - Err(error) => { - let error = cleanup(&mut child, error); - drop(join_reader("stdout", stdout_reader)); - return Err(error); - } - }; - let status = wait_for_child(&mut child, deadline); - let stdout = join_reader("stdout", stdout_reader); - let stderr = join_reader("stderr", stderr_reader); - let status = status?; - let stdout = stdout?; - let stderr = stderr?; - refuse_exceeded("stdout", &stdout)?; - refuse_exceeded("stderr", &stderr)?; - Ok(ProcessOutput { - succeeded: status.success(), - stdout: stdout.bytes, - stderr: stderr.bytes, - }) -} - -fn wait_for_child( - child: &mut Child, - deadline: Option, -) -> Result { - let Some(duration) = deadline else { - return match child.wait() { - Ok(status) => Ok(status), - Err(source) => Err(cleanup( - child, - ProcessError::Io { - action: "wait", - source, - }, - )), - }; - }; - let Some(expires) = Instant::now().checked_add(duration) else { - return Err(cleanup(child, ProcessError::Timeout(duration))); - }; - loop { - match child.try_wait() { - Err(source) => { - return Err(cleanup( - child, - ProcessError::Io { - action: "poll", - source, - }, - )); - } - Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() >= expires => { - return Err(cleanup(child, ProcessError::Timeout(duration))); - } - Ok(None) => thread::sleep(Duration::from_millis(10)), - } - } -} - -fn start_reader( - stream: &'static str, - reader: impl io::Read + Send + 'static, -) -> Result>, ProcessError> { - thread::Builder::new() - .name(format!("fuzz-{stream}-reader")) - .spawn(move || bounded_bytes(reader, OUTPUT_LIMIT)) - .map_err(|source| ProcessError::Io { - action: "start output reader", - source, - }) -} - -fn join_reader( - stream: &'static str, - worker: JoinHandle>, -) -> Result { - worker - .join() - .map_err(|_panic| ProcessError::ReaderPanic(stream))? - .map_err(|source| ProcessError::Io { - action: "read child output", - source, - }) -} - -const fn refuse_exceeded(stream: &'static str, output: &BoundedBytes) -> Result<(), ProcessError> { - if output.exceeded { - Err(ProcessError::OutputLimit { - stream, - maximum: OUTPUT_LIMIT, - }) - } else { - Ok(()) - } -} - -fn cleanup(child: &mut std::process::Child, primary: ProcessError) -> ProcessError { - let kill = child.kill(); - let wait = child.wait(); - if let Err(source) = kill - && source.kind() != io::ErrorKind::InvalidInput - { - return ProcessError::Cleanup { - primary: Box::new(primary), - action: "kill", - source, - }; - } - if let Err(source) = wait { - return ProcessError::Cleanup { - primary: Box::new(primary), - action: "wait", - source, - }; - } - primary -} diff --git a/xtask/src/fuzz_campaign/process/error.rs b/xtask/src/fuzz_campaign/process/error.rs deleted file mode 100644 index 2308a69..0000000 --- a/xtask/src/fuzz_campaign/process/error.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! This module owns typed cargo-fuzz process failures. - -use std::error::Error; -use std::fmt; -use std::io; -use std::time::Duration; - -pub(crate) enum ProcessError { - Cleanup { - primary: Box, - action: &'static str, - source: io::Error, - }, - Io { - action: &'static str, - source: io::Error, - }, - MissingStream(&'static str), - OutputLimit { - stream: &'static str, - maximum: usize, - }, - ReaderPanic(&'static str), - Timeout(Duration), -} - -impl fmt::Debug for ProcessError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, formatter) - } -} - -impl fmt::Display for ProcessError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Cleanup { - primary, action, .. - } => write!( - formatter, - "{primary}; additionally failed to {action} child process" - ), - Self::Io { action, .. } => write!(formatter, "cannot {action} cargo-fuzz process"), - Self::MissingStream(stream) => { - write!(formatter, "cargo-fuzz {stream} pipe is unavailable") - } - Self::OutputLimit { stream, maximum } => { - write!( - formatter, - "cargo-fuzz {stream} exceeds the {maximum}-byte bound" - ) - } - Self::ReaderPanic(stream) => write!(formatter, "cargo-fuzz {stream} reader panicked"), - Self::Timeout(duration) => write!( - formatter, - "cargo-fuzz process exceeded its {}-second deadline", - duration.as_secs() - ), - } - } -} - -impl Error for ProcessError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), - Self::MissingStream(_) - | Self::OutputLimit { .. } - | Self::ReaderPanic(_) - | Self::Timeout(_) => None, - } - } -} diff --git a/xtask/src/fuzz_campaign/target.rs b/xtask/src/fuzz_campaign/target.rs index c3f5837..ff92e67 100644 --- a/xtask/src/fuzz_campaign/target.rs +++ b/xtask/src/fuzz_campaign/target.rs @@ -13,7 +13,9 @@ use std::process::Command; pub(super) use error::TargetError; use super::policy::CampaignPolicy; -use super::process; +use crate::bounded_process; + +const CARGO_FUZZ_PROCESS: &str = "cargo-fuzz"; #[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] pub(super) struct FuzzTarget(String); @@ -48,7 +50,7 @@ pub(super) fn registered( .arg(format!("+{}", policy.toolchain())) .args(["fuzz", "list"]) .current_dir(repository_root); - let output = process::capture(&mut command, None)?; + let output = bounded_process::capture(CARGO_FUZZ_PROCESS, &mut command, None)?; if !output.succeeded { return Err(TargetError::ListFailed { stdout: output.stdout, diff --git a/xtask/src/fuzz_campaign/target/error.rs b/xtask/src/fuzz_campaign/target/error.rs index 8c7c74f..7ea517c 100644 --- a/xtask/src/fuzz_campaign/target/error.rs +++ b/xtask/src/fuzz_campaign/target/error.rs @@ -5,8 +5,8 @@ use std::fmt; use std::io; use std::path::PathBuf; +use crate::bounded_process::ProcessError; use crate::diagnostic::{escaped_controls, escaped_path}; -use crate::fuzz_campaign::process::ProcessError; pub(crate) enum TargetError { Disagreement { diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index f88ebb1..03518e1 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -29,6 +29,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "fast_cdc", "golden_protocol", "layout_record", + "repository_json", "segment_format", ] ); diff --git a/xtask/src/fuzz_campaign/workflow_tests.rs b/xtask/src/fuzz_campaign/workflow_tests.rs index 76456fe..8e776da 100644 --- a/xtask/src/fuzz_campaign/workflow_tests.rs +++ b/xtask/src/fuzz_campaign/workflow_tests.rs @@ -3,6 +3,8 @@ use std::error::Error; use std::fs; use std::path::Path; +use yaml_rust2::{Yaml, YamlLoader}; + const CI: &str = include_str!("../../../.github/workflows/ci.yml"); const SCHEDULED: &str = include_str!("../../../.github/workflows/fuzz-scheduled.yml"); @@ -65,24 +67,95 @@ fn only_successfully_minimized_corpora_are_retained() -> Result<(), Box Result<(), Box> { - let (_, ci_fuzz) = CI - .split_once(" fuzz-smoke:\n") - .ok_or("CI has no fuzz-smoke job")?; - let (ci_fuzz, _) = ci_fuzz - .split_once("\n dependency-policy:") - .ok_or("CI fuzz-smoke job has no closing job")?; - for workflow in [ci_fuzz, SCHEDULED] { - assert!(!workflow.contains("python")); - assert!(!workflow.contains(".py")); - assert!(workflow.contains("cargo xtask fuzz github-env")); - assert!(workflow.contains("cargo xtask fuzz run")); + for (workflow, job) in [(CI, "fuzz-smoke"), (SCHEDULED, "fuzz")] { + workflow_delegates_to_xtask(workflow, job)?; + } + let scheduled_commands = run_commands(SCHEDULED, "fuzz")?; + if command_index(&scheduled_commands, "cargo xtask fuzz check-corpus").is_none() { + return Err("scheduled workflow does not validate its corpus".into()); + } + if command_index(&scheduled_commands, "cargo xtask fuzz minimize").is_none() { + return Err("scheduled workflow does not minimize its corpus".into()); } - assert!(SCHEDULED.contains("cargo xtask fuzz check-corpus")); - assert!(SCHEDULED.contains("cargo xtask fuzz build")); - assert!(SCHEDULED.contains("cargo xtask fuzz minimize")); Ok(()) } +#[test] +fn non_run_scalar_cannot_impersonate_the_fuzz_build_step() { + let disguised = CI.replace( + " run: cargo xtask fuzz build --profile smoke", + " env:\n BUILD_NOTE: cargo xtask fuzz build --profile smoke", + ); + + assert!(workflow_delegates_to_xtask(&disguised, "fuzz-smoke").is_err()); +} + +fn workflow_delegates_to_xtask(workflow: &str, job: &str) -> Result<(), Box> { + let commands = run_commands(workflow, job)?; + if commands + .iter() + .any(|command| command.contains("python") || command.contains(".py")) + { + return Err("fuzz workflow delegates to Python".into()); + } + if command_index(&commands, "cargo xtask fuzz github-env").is_none() { + return Err("fuzz workflow does not load xtask policy".into()); + } + let build = command_index(&commands, "cargo xtask fuzz build") + .ok_or("fuzz workflow does not build targets")?; + let run = command_index(&commands, "cargo xtask fuzz run") + .ok_or("fuzz workflow does not run targets")?; + if build >= run { + return Err("fuzz workflow does not build before running targets".into()); + } + Ok(()) +} + +fn run_commands(workflow: &str, job: &str) -> Result, Box> { + let documents = YamlLoader::load_from_str(workflow)?; + let [document] = documents.as_slice() else { + return Err("fuzz workflow must contain exactly one YAML document".into()); + }; + let jobs = mapping_field(document, "jobs") + .and_then(Yaml::as_hash) + .ok_or("fuzz workflow jobs must be a mapping")?; + let job = jobs + .get(&Yaml::String(job.to_owned())) + .ok_or("fuzz workflow has no reviewed fuzz job")?; + let steps = mapping_field(job, "steps") + .and_then(Yaml::as_vec) + .ok_or("fuzz workflow job steps must be a sequence")?; + let mut commands = Vec::new(); + for step in steps { + let Some(run) = mapping_field(step, "run") else { + continue; + }; + let run = run + .as_str() + .ok_or("fuzz workflow run step must be a string")?; + commands.push(run.to_owned()); + } + Ok(commands) +} + +fn mapping_field<'a>(node: &'a Yaml, field: &str) -> Option<&'a Yaml> { + node.as_hash()?.get(&Yaml::String(field.to_owned())) +} + +fn command_index(commands: &[String], expected: &str) -> Option { + commands.iter().position(|command| { + command.lines().any(|line| { + line.trim().strip_prefix(expected).is_some_and(|suffix| { + suffix.is_empty() + || suffix + .as_bytes() + .first() + .is_some_and(u8::is_ascii_whitespace) + }) + }) + }) +} + fn checkout_references(workflow: &str) -> BTreeSet<&str> { action_references(workflow) .filter_map(|(action, reference)| (action == "actions/checkout").then_some(reference)) diff --git a/xtask/src/git_inventory.rs b/xtask/src/git_inventory.rs new file mode 100644 index 0000000..e1df1d5 --- /dev/null +++ b/xtask/src/git_inventory.rs @@ -0,0 +1,9 @@ +//! This module owns bounded, deterministic Git path inventory. + +mod error; +mod path_stream; +mod process; + +pub(crate) use error::{GitInventoryError, GitOutputUnit}; +pub(crate) use path_stream::GitPath; +pub(crate) use process::paths_with; diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs new file mode 100644 index 0000000..f15ce42 --- /dev/null +++ b/xtask/src/git_inventory/error.rs @@ -0,0 +1,161 @@ +//! This module owns typed Git inventory failures and stable diagnostics. + +use std::error::Error; +use std::fmt::{self, Write as _}; +use std::io; +use std::string::FromUtf8Error; + +use crate::bounded_process::ProcessError; +use crate::diagnostic::escaped_controls; + +#[derive(Clone, Copy)] +/// The unit named by a bounded Git-output failure. +pub(crate) enum GitOutputUnit { + /// A byte-count bound. + Bytes, + /// A path-record count bound. + Items, +} + +/// A typed failure while listing or decoding repository paths from Git. +pub(crate) enum GitInventoryError { + /// Git emitted the same path record more than once. + DuplicatePath(Vec), + /// Git emitted an empty NUL-framed path record. + EmptyPath { operation: &'static str }, + /// Git exited unsuccessfully and returned valid UTF-8 diagnostics. + Failed { + operation: &'static str, + code: Option, + stderr: String, + }, + /// Git exited unsuccessfully with diagnostics that were not UTF-8. + DiagnosticEncoding { + operation: &'static str, + code: Option, + source: FromUtf8Error, + }, + /// A retained byte or item count exceeded its fixed bound. + OutputBound { + operation: &'static str, + stream: &'static str, + maximum: usize, + unit: GitOutputUnit, + }, + /// Git ended its output with bytes not terminated by a NUL delimiter. + OutputFraming { operation: &'static str }, + /// Deadline-bounded Git process execution failed. + Process { + operation: &'static str, + source: ProcessError, + }, + /// A named operating-system action for the Git child failed. + Run { + operation: &'static str, + action: &'static str, + source: io::Error, + }, +} + +impl fmt::Debug for GitInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for GitInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DuplicatePath(path) => { + formatter.write_str("git returned duplicate path `")?; + escaped_bytes(formatter, path)?; + formatter.write_str("`") + } + Self::EmptyPath { operation } => { + write!(formatter, "`{operation}` returned an empty path") + } + Self::Failed { + operation, + code, + stderr, + } => git_failed(formatter, operation, *code, stderr), + Self::DiagnosticEncoding { + operation, code, .. + } => write!( + formatter, + "`{operation}` failed with code {code:?} and returned non-UTF-8 diagnostics" + ), + Self::OutputBound { + operation, + stream, + maximum, + unit, + } => write!( + formatter, + "`{operation}` exceeded the {stream} bound of {maximum} {}", + unit.label() + ), + Self::OutputFraming { operation } => { + write!( + formatter, + "`{operation}` returned a non-NUL-terminated path" + ) + } + Self::Process { operation, source } => { + write!(formatter, "{source} while running `{operation}`") + } + Self::Run { + operation, action, .. + } => write!(formatter, "cannot {action} `{operation}`"), + } + } +} + +impl GitOutputUnit { + const fn label(self) -> &'static str { + match self { + Self::Bytes => "bytes", + Self::Items => "items", + } + } +} + +impl Error for GitInventoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::DiagnosticEncoding { source, .. } => Some(source), + Self::Process { source, .. } => Some(source), + Self::Run { source, .. } => Some(source), + Self::DuplicatePath(_) + | Self::EmptyPath { .. } + | Self::Failed { .. } + | Self::OutputBound { .. } + | Self::OutputFraming { .. } => None, + } + } +} + +fn git_failed( + formatter: &mut fmt::Formatter<'_>, + operation: &str, + code: Option, + stderr: &str, +) -> fmt::Result { + write!(formatter, "`{operation}` failed with code {code:?}")?; + if stderr.is_empty() { + return Ok(()); + } + formatter.write_str(": ")?; + escaped_controls(formatter, stderr.trim_end()) +} + +fn escaped_bytes(formatter: &mut fmt::Formatter<'_>, bytes: &[u8]) -> fmt::Result { + for byte in bytes { + if byte.is_ascii_graphic() || *byte == b' ' { + formatter.write_char(char::from(*byte))?; + } else { + write!(formatter, "\\x{byte:02x}")?; + } + } + Ok(()) +} diff --git a/xtask/src/source_structure/git_path_stream.rs b/xtask/src/git_inventory/path_stream.rs similarity index 55% rename from xtask/src/source_structure/git_path_stream.rs rename to xtask/src/git_inventory/path_stream.rs index 50d1ba0..cb4597b 100644 --- a/xtask/src/source_structure/git_path_stream.rs +++ b/xtask/src/git_inventory/path_stream.rs @@ -3,11 +3,14 @@ use std::collections::BTreeSet; use std::io::Read; -use super::{GitOutputUnit, SourceStructureError}; +use super::{GitInventoryError, GitOutputUnit}; + +/// Maximum retained bytes in one complete Git path stream. +pub(super) const GIT_PATH_STREAM_LIMIT_BYTES: usize = 16_777_216; const GIT_PATH_LIMITS: GitPathLimits = GitPathLimits { path_bytes: 4_096, - stream_bytes: 16_777_216, + stream_bytes: GIT_PATH_STREAM_LIMIT_BYTES, paths: 100_000, }; @@ -18,10 +21,15 @@ struct GitPathLimits { paths: usize, } +/// Decodes Git's NUL-framed path output into deterministic bytewise order. +/// +/// The call blocks while reading and refuses empty, duplicate, unterminated, +/// oversized, or excessive path records. Individual paths are limited to 4,096 +/// bytes; the stream is limited to 16 MiB and 100,000 records. pub(super) fn read_paths( reader: impl Read, operation: &'static str, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { read_paths_with(reader, operation, GIT_PATH_LIMITS) } @@ -29,13 +37,13 @@ fn read_paths_with( mut reader: impl Read, operation: &'static str, limits: GitPathLimits, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { let mut decoder = GitPathDecoder::new(operation, limits); let mut buffer = [0_u8; 4_096]; loop { let read = reader .read(&mut buffer) - .map_err(|source| SourceStructureError::RunGit { + .map_err(|source| GitInventoryError::Run { operation, action: "read paths from", source, @@ -43,14 +51,12 @@ fn read_paths_with( if read == 0 { break; } - let bytes = buffer - .get(..read) - .ok_or(SourceStructureError::GitOutputBound { - operation, - stream: "path read", - maximum: buffer.len(), - unit: GitOutputUnit::Bytes, - })?; + let bytes = buffer.get(..read).ok_or(GitInventoryError::OutputBound { + operation, + stream: "path read", + maximum: buffer.len(), + unit: GitOutputUnit::Bytes, + })?; decoder.admit(bytes)?; } decoder.finish() @@ -62,18 +68,24 @@ struct GitPathDecoder { observed_bytes: usize, observed_paths: usize, operation: &'static str, - paths: BTreeSet, + paths: BTreeSet, } #[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] -pub(super) struct GitPathRecord(Vec); - -impl GitPathRecord { - pub(super) const fn new(bytes: Vec) -> Self { +/// An opaque repository path admitted from Git's byte-oriented output. +/// +/// Ordering is bytewise and therefore independent of locale and filesystem +/// enumeration order. +pub(crate) struct GitPath(Vec); + +impl GitPath { + /// Wraps path bytes that the stream decoder has already admitted. + pub(crate) const fn new(bytes: Vec) -> Self { Self(bytes) } - pub(super) fn as_bytes(&self) -> &[u8] { + /// Returns the exact repository-relative bytes reported by Git. + pub(crate) fn as_bytes(&self) -> &[u8] { &self.0 } } @@ -90,17 +102,18 @@ impl GitPathDecoder { } } - fn admit(&mut self, bytes: &[u8]) -> Result<(), SourceStructureError> { - self.observed_bytes = self.observed_bytes.checked_add(bytes.len()).ok_or( - SourceStructureError::GitOutputBound { - operation: self.operation, - stream: "path stream bytes", - maximum: self.limits.stream_bytes, - unit: GitOutputUnit::Bytes, - }, - )?; + fn admit(&mut self, bytes: &[u8]) -> Result<(), GitInventoryError> { + self.observed_bytes = + self.observed_bytes + .checked_add(bytes.len()) + .ok_or(GitInventoryError::OutputBound { + operation: self.operation, + stream: "path stream bytes", + maximum: self.limits.stream_bytes, + unit: GitOutputUnit::Bytes, + })?; if self.observed_bytes > self.limits.stream_bytes { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path stream bytes", maximum: self.limits.stream_bytes, @@ -113,12 +126,12 @@ impl GitPathDecoder { Ok(()) } - fn admit_byte(&mut self, byte: u8) -> Result<(), SourceStructureError> { + fn admit_byte(&mut self, byte: u8) -> Result<(), GitInventoryError> { if byte == 0 { return self.admit_path(); } if self.current.len() >= self.limits.path_bytes { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path bytes", maximum: self.limits.path_bytes, @@ -129,44 +142,42 @@ impl GitPathDecoder { Ok(()) } - fn admit_path(&mut self) -> Result<(), SourceStructureError> { + fn admit_path(&mut self) -> Result<(), GitInventoryError> { if self.current.is_empty() { - return Err(SourceStructureError::GitOutputFraming { + return Err(GitInventoryError::EmptyPath { operation: self.operation, }); } self.observed_paths = self.observed_paths .checked_add(1) - .ok_or(SourceStructureError::GitOutputBound { + .ok_or(GitInventoryError::OutputBound { operation: self.operation, stream: "path count", maximum: self.limits.paths, unit: GitOutputUnit::Items, })?; if self.observed_paths > self.limits.paths { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path count", maximum: self.limits.paths, unit: GitOutputUnit::Items, }); } - let path = GitPathRecord::new(std::mem::take(&mut self.current)); + let path = GitPath::new(std::mem::take(&mut self.current)); if self.paths.insert(path.clone()) { Ok(()) } else { - Err(SourceStructureError::DuplicatePath( - path.as_bytes().to_vec(), - )) + Err(GitInventoryError::DuplicatePath(path.as_bytes().to_vec())) } } - fn finish(self) -> Result, SourceStructureError> { + fn finish(self) -> Result, GitInventoryError> { if self.current.is_empty() { Ok(self.paths) } else { - Err(SourceStructureError::GitOutputFraming { + Err(GitInventoryError::OutputFraming { operation: self.operation, }) } @@ -174,5 +185,5 @@ impl GitPathDecoder { } #[cfg(test)] -#[path = "git_path_stream/tests.rs"] +#[path = "path_stream/tests.rs"] mod tests; diff --git a/xtask/src/source_structure/git_path_stream/tests.rs b/xtask/src/git_inventory/path_stream/tests.rs similarity index 85% rename from xtask/src/source_structure/git_path_stream/tests.rs rename to xtask/src/git_inventory/path_stream/tests.rs index 29ead53..d7a86f4 100644 --- a/xtask/src/source_structure/git_path_stream/tests.rs +++ b/xtask/src/git_inventory/path_stream/tests.rs @@ -2,7 +2,7 @@ use std::io::Cursor; -use super::{GitPathLimits, SourceStructureError, read_paths_with}; +use super::{GitInventoryError, GitPathLimits, read_paths_with}; const TEST_LIMITS: GitPathLimits = GitPathLimits { path_bytes: 3, @@ -15,7 +15,7 @@ fn git_path_stream_refuses_an_oversized_path_without_buffering_the_tail() { let result = read_paths_with(Cursor::new(b"abcd\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path bytes", maximum: 3, .. @@ -28,7 +28,7 @@ fn git_path_stream_refuses_an_oversized_inventory() { let result = read_paths_with(Cursor::new(b"a\0b\0c\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path count", maximum: 2, .. @@ -41,7 +41,7 @@ fn git_path_stream_refuses_excess_framing_bytes() { let result = read_paths_with(Cursor::new(b"\0\0\0\0\0\0\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path stream bytes", maximum: 6, .. @@ -54,7 +54,7 @@ fn git_path_stream_requires_nul_framing() { let result = read_paths_with(Cursor::new(b"abc"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputFraming { + Err(GitInventoryError::OutputFraming { operation: "test paths" }) )); @@ -65,11 +65,15 @@ fn git_path_stream_refuses_empty_records() { for stream in [&b"\0"[..], &b"a\0\0"[..]] { let result = read_paths_with(Cursor::new(stream), "test paths", TEST_LIMITS); assert!(matches!( - result, - Err(SourceStructureError::GitOutputFraming { + &result, + Err(GitInventoryError::EmptyPath { operation: "test paths" }) )); + assert_eq!( + result.err().map(|error| error.to_string()), + Some(String::from("`test paths` returned an empty path")) + ); } } @@ -78,7 +82,7 @@ fn git_path_stream_refuses_duplicate_records() { let result = read_paths_with(Cursor::new(b"a\0a\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::DuplicatePath(ref path)) if path == b"a" + Err(GitInventoryError::DuplicatePath(ref path)) if path == b"a" )); } diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs new file mode 100644 index 0000000..9ce7036 --- /dev/null +++ b/xtask/src/git_inventory/process.rs @@ -0,0 +1,129 @@ +//! This module owns deadline-bounded Git path process execution. + +use std::collections::BTreeSet; +use std::env; +use std::ffi::OsStr; +use std::io; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use crate::bounded_process::{self, CaptureLimits, ProcessError, ProcessOutput}; + +use super::path_stream::{GIT_PATH_STREAM_LIMIT_BYTES, GitPath, read_paths}; +use super::{GitInventoryError, GitOutputUnit}; + +const GIT_DEADLINE: Duration = Duration::from_mins(2); +const GIT_DIAGNOSTIC_LIMIT_BYTES: usize = 65_536; +const GIT_CAPTURE_LIMITS: CaptureLimits = + CaptureLimits::new(GIT_PATH_STREAM_LIMIT_BYTES, GIT_DIAGNOSTIC_LIMIT_BYTES); + +/// Lists paths through an injected capability-bound spawn operation. +/// +/// The adapter materializes at most the 16 MiB path-stream bound before +/// deterministic NUL-framed decoding. +pub(crate) fn paths_with( + arguments: &[&str], + operation: &'static str, + spawn: impl FnOnce(&mut Command) -> Result, +) -> Result, GitInventoryError> { + paths_with_deadline(arguments, operation, GIT_DEADLINE, spawn) +} + +fn paths_with_deadline( + arguments: &[&str], + operation: &'static str, + deadline: Duration, + spawn: impl FnOnce(&mut Command) -> Result, +) -> Result, GitInventoryError> { + let path = env::var_os("PATH").ok_or_else(|| GitInventoryError::Run { + operation, + action: "read PATH for", + source: io::Error::new(io::ErrorKind::NotFound, "PATH is unavailable"), + })?; + let mut command = git_command(arguments, &path); + let output = bounded_process::capture_with_limits( + "git", + &mut command, + Some(deadline), + GIT_CAPTURE_LIMITS, + spawn, + ) + .map_err(|source| process_failure(operation, source))?; + inventory_output(operation, output) +} + +fn inventory_output( + operation: &'static str, + output: ProcessOutput, +) -> Result, GitInventoryError> { + if !output.succeeded { + return Err(git_failure(operation, output.code, output.stderr)); + } + read_paths(output.stdout.as_slice(), operation) +} + +fn git_command(arguments: &[&str], path: &OsStr) -> Command { + let mut command = Command::new("git"); + command + .args(arguments) + .stdin(Stdio::null()) + .env_clear() + .env("PATH", path) + .env("LC_ALL", "C") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .env("GIT_OPTIONAL_LOCKS", "0"); + command +} + +fn process_failure(operation: &'static str, source: ProcessError) -> GitInventoryError { + match source { + ProcessError::OutputLimit { + stream: "stdout", + maximum, + .. + } => output_bound(operation, "path stream bytes", maximum), + ProcessError::OutputLimit { + stream: "stderr", + maximum, + .. + } => output_bound(operation, "diagnostic bytes", maximum), + source => GitInventoryError::Process { operation, source }, + } +} + +const fn output_bound( + operation: &'static str, + stream: &'static str, + maximum: usize, +) -> GitInventoryError { + GitInventoryError::OutputBound { + operation, + stream, + maximum, + unit: GitOutputUnit::Bytes, + } +} + +fn git_failure( + operation: &'static str, + code: Option, + diagnostic: Vec, +) -> GitInventoryError { + match String::from_utf8(diagnostic) { + Ok(stderr) => GitInventoryError::Failed { + operation, + code, + stderr, + }, + Err(source) => GitInventoryError::DiagnosticEncoding { + operation, + code, + source, + }, + } +} + +#[cfg(test)] +mod tests; diff --git a/xtask/src/git_inventory/process/tests.rs b/xtask/src/git_inventory/process/tests.rs new file mode 100644 index 0000000..d9b3c89 --- /dev/null +++ b/xtask/src/git_inventory/process/tests.rs @@ -0,0 +1,149 @@ +//! This module owns Git process deadline and diagnostic regression evidence. + +use std::collections::BTreeMap; +use std::env; +use std::ffi::OsString; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use std::os::unix::process::CommandExt; + +use super::{ + GIT_DIAGNOSTIC_LIMIT_BYTES, git_command, git_failure, inventory_output, paths_with_deadline, + process_failure, +}; +use crate::bounded_process::{ProcessError, ProcessOutput}; +use crate::git_inventory::GitInventoryError; + +const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_GIT_CHILD"; +const PARKED_CHILD_TEST: &str = "git_inventory::process::tests::process_child_parks_indefinitely"; + +#[test] +fn git_inventory_receives_only_reviewed_environment() { + let command = git_command(&["status"], OsString::from("/reviewed/tools").as_os_str()); + let observed = command + .get_envs() + .map(|(name, value)| (name.to_owned(), value.map(OsString::from))) + .collect::>(); + let expected = BTreeMap::from([ + ( + OsString::from("GIT_CONFIG_COUNT"), + Some(OsString::from("0")), + ), + ( + OsString::from("GIT_CONFIG_GLOBAL"), + Some(OsString::from("/dev/null")), + ), + ( + OsString::from("GIT_CONFIG_NOSYSTEM"), + Some(OsString::from("1")), + ), + ( + OsString::from("GIT_OPTIONAL_LOCKS"), + Some(OsString::from("0")), + ), + (OsString::from("LC_ALL"), Some(OsString::from("C"))), + ( + OsString::from("PATH"), + Some(OsString::from("/reviewed/tools")), + ), + ]); + + assert_eq!(observed, expected); +} + +#[test] +fn git_diagnostic_encoding_failure_retains_exit_status() { + let error = git_failure("test diagnostics", Some(9), vec![u8::MAX]); + assert!(matches!( + error, + GitInventoryError::DiagnosticEncoding { + operation: "test diagnostics", + code: Some(9), + .. + } + )); +} + +#[test] +fn failed_git_status_precedes_stdout_decoding() { + let result = inventory_output( + "test failure precedence", + ProcessOutput { + code: Some(23), + succeeded: false, + stdout: b"unterminated".to_vec(), + stderr: b"fatal: reviewed failure\n".to_vec(), + }, + ); + + assert!(matches!( + result, + Err(GitInventoryError::Failed { + operation: "test failure precedence", + code: Some(23), + ref stderr, + }) if stderr == "fatal: reviewed failure\n" + )); +} + +#[test] +fn git_diagnostic_limit_maps_to_the_inventory_boundary() { + let error = process_failure( + "test diagnostics", + ProcessError::OutputLimit { + program: "git", + stream: "stderr", + maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, + }, + ); + assert!(matches!( + error, + GitInventoryError::OutputBound { + operation: "test diagnostics", + stream: "diagnostic bytes", + maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, + .. + } + )); +} + +#[test] +fn stalled_git_process_obeys_the_inventory_deadline() -> Result<(), Box> { + let executable = env::current_exe()?; + let child = Command::new(executable) + .args(["--exact", PARKED_CHILD_TEST]) + .env(PARKED_CHILD, "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn()?; + let deadline = Duration::from_millis(25); + + let result = paths_with_deadline(&[], "test Git inventory", deadline, move |_command| { + Ok(child) + }); + + assert!(matches!( + result, + Err(GitInventoryError::Process { + operation: "test Git inventory", + source: ProcessError::Timeout { + program: "git", + duration, + }, + }) if duration == deadline + )); + Ok(()) +} + +#[test] +fn process_child_parks_indefinitely() { + if env::var_os(PARKED_CHILD).is_none() { + return; + } + loop { + std::thread::park(); + } +} diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index e0d3ddd..dfd4e8c 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -16,6 +16,14 @@ mod diagnostic; )] mod golden_protocol_fuzz; +#[cfg(feature = "repository-json-fuzz")] +#[allow( + clippy::redundant_pub_crate, + reason = "the library facade deliberately hides the repository parser implementation" +)] +#[path = "documentation_integrity/node_toolchain/unique_json.rs"] +mod repository_json; + pub mod protocol_admission; #[cfg(test)] @@ -50,3 +58,34 @@ pub fn admit_golden_protocol(selector: u8, input: &[u8]) -> GoldenProtocolAdmiss GoldenProtocolAdmission::Refused } } + +/// Whether bounded duplicate-refusing repository JSON admission accepted input. +#[cfg(feature = "repository-json-fuzz")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RepositoryJsonAdmission { + /// UTF-8 JSON within the byte and nesting limits was admitted. + Admitted, + /// Encoding, syntax, size, nesting, or duplicate members were refused. + Refused, +} + +/// Exercises the exact duplicate-refusing repository JSON parser. +/// +/// Input above one mebibyte is refused before UTF-8 decoding or recursive +/// parsing. No serializer-owned value escapes this fuzz-only boundary. +#[cfg(feature = "repository-json-fuzz")] +#[must_use] +pub fn admit_repository_json(input: &[u8]) -> RepositoryJsonAdmission { + const MAXIMUM_BYTES: usize = 1_048_576; + if input.len() > MAXIMUM_BYTES { + return RepositoryJsonAdmission::Refused; + } + let Ok(raw) = std::str::from_utf8(input) else { + return RepositoryJsonAdmission::Refused; + }; + if repository_json::parse(raw).is_ok() { + RepositoryJsonAdmission::Admitted + } else { + RepositoryJsonAdmission::Refused + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2635b98..a6d4358 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -8,17 +8,32 @@ reason = "the command dispatcher owns this private repository task" )] mod benchmark_baseline; +#[allow( + clippy::redundant_pub_crate, + reason = "bounded process execution is shared by sibling repository tasks" +)] +mod bounded_process; mod diagnostic; #[allow( clippy::redundant_pub_crate, reason = "the command and task-error boundaries are sibling consumers" )] +mod documentation_integrity; +#[allow( + clippy::redundant_pub_crate, + reason = "the command and task-error boundaries are sibling consumers" +)] mod fuzz_campaign; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" )] mod fuzz_seed_corpus; +#[allow( + clippy::redundant_pub_crate, + reason = "bounded Git inventory is shared by sibling repository tasks" +)] +mod git_inventory; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" @@ -34,6 +49,17 @@ mod process_output; reason = "the command and task-error boundaries are sibling consumers" )] mod protocol_conformance; +#[allow( + clippy::redundant_pub_crate, + reason = "repository file admission is shared by sibling repository tasks" +)] +mod repository_file; +#[cfg(test)] +#[allow( + clippy::redundant_pub_crate, + reason = "repository fixture execution is shared by sibling test modules" +)] +mod repository_fixture; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" @@ -44,10 +70,9 @@ mod source_structure; reason = "the parent command dispatcher is the only consumer" )] mod task_error; -#[cfg(test)] #[allow( clippy::redundant_pub_crate, - reason = "scoped test directories are shared by sibling test modules" + reason = "scoped evidence directories are shared by verification and test modules" )] mod test_directory; @@ -89,6 +114,12 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { "conformance-check" => { protocol_conformance::check(repository_root)?; } + "documentation-integrity-check" => { + documentation_integrity::check(repository_root)?; + } + "documentation-refusal-check" => { + documentation_integrity::check_refusals()?; + } "prepare-fuzz-corpus" => { fuzz_seed_corpus::prepare(repository_root)?; } @@ -99,6 +130,7 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { source_structure::check(repository_root)?; } "verify" => { + documentation_integrity::check(repository_root)?; golden_file_worldline::check(repository_root)?; protocol_conformance::check(repository_root)?; source_structure::check(repository_root)?; diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs new file mode 100644 index 0000000..7bdd168 --- /dev/null +++ b/xtask/src/repository_file.rs @@ -0,0 +1,211 @@ +//! This module owns capability-relative, no-follow repository-file admission. +//! +//! Repository tasks are intentionally supported only on Unix hosts. This module +//! binds an opened repository root to Unix device and inode identity so that path +//! replacement cannot silently redirect a read. Supporting another host requires +//! an equivalent stable directory-identity contract before enabling these tasks. + +mod exact_copy; + +use std::fs::File; +use std::io; +use std::os::fd::{OwnedFd, RawFd}; +use std::os::unix::fs::MetadataExt as UnixMetadataExt; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; + +use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::ambient_authority; +use cap_std::fs::{Dir, OpenOptions}; +use repository_process_spawn::set_working_directory; + +pub(crate) use exact_copy::copy_exact; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReadAccessPolicy { + Enabled, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum BlockingIoPolicy { + Refuse, +} + +#[derive(Clone, Copy)] +pub(crate) struct RepositoryReadPolicy { + read_access: ReadAccessPolicy, + blocking_io: BlockingIoPolicy, +} + +pub(crate) const REPOSITORY_READ_POLICY: RepositoryReadPolicy = RepositoryReadPolicy { + read_access: ReadAccessPolicy::Enabled, + blocking_io: BlockingIoPolicy::Refuse, +}; + +/// First descriptor outside the standard stdin, stdout, and stderr range. +const CHILD_DESCRIPTOR_MINIMUM: RawFd = 3; + +pub(crate) struct RepositoryRoot { + directory: Dir, + identity: DirectoryIdentity, + path: PathBuf, +} + +/// An exact child-process working directory backed by an owned descriptor. +/// +/// Child setup changes directory through this descriptor, never by reopening +/// the ambient repository path. +pub(crate) struct RepositoryProcessDirectory { + directory: OwnedFd, +} + +#[derive(Eq, PartialEq)] +struct DirectoryIdentity { + device: u64, + inode: u64, +} + +#[derive(Eq, PartialEq)] +pub(crate) struct RepositoryFileIdentity { + device: u64, + inode: u64, + bytes: u64, + modified_seconds: i64, + modified_nanoseconds: i64, + changed_seconds: i64, + changed_nanoseconds: i64, +} + +pub(crate) enum OpenRepositoryFileError { + Io(io::Error), + NonRegular, +} + +impl RepositoryFileIdentity { + pub(crate) fn read(file: &File) -> Result { + Ok(Self::from(&file.metadata()?)) + } + + pub(crate) const fn bytes(&self) -> u64 { + self.bytes + } +} + +impl RepositoryRoot { + pub(crate) fn open(path: &Path) -> Result { + let directory = Dir::open_ambient_dir(path, ambient_authority())?; + let identity = DirectoryIdentity::from(&directory.dir_metadata()?); + Ok(Self { + directory, + identity, + path: path.to_owned(), + }) + } + + pub(crate) fn display_path(&self, relative: &Path) -> PathBuf { + self.path.join(relative) + } + + pub(crate) fn is_current_path(&self) -> Result { + let current = Dir::open_ambient_dir(&self.path, ambient_authority())?; + let identity = DirectoryIdentity::from(¤t.dir_metadata()?); + Ok(self.identity == identity) + } + + /// Returns an exact child-process handle for this opened directory. + pub(crate) fn process_directory(&self) -> Result { + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, CHILD_DESCRIPTOR_MINIMUM)?; + Ok(RepositoryProcessDirectory { directory }) + } + + pub(crate) fn open_file(&self, relative: &Path) -> Result { + let file = match self + .directory + .open_with(relative, &REPOSITORY_READ_POLICY.options()) + { + Ok(file) => file.into_std(), + Err(source) => { + if self + .directory + .symlink_metadata(relative) + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(OpenRepositoryFileError::NonRegular); + } + return Err(OpenRepositoryFileError::Io(source)); + } + }; + let metadata = file.metadata().map_err(OpenRepositoryFileError::Io)?; + if metadata.is_file() { + Ok(file) + } else { + Err(OpenRepositoryFileError::NonRegular) + } + } +} + +impl RepositoryProcessDirectory { + /// Starts a child from the exact opened repository directory. + /// + /// The child changes directory through its retained descriptor after fork + /// and before exec. Parent process state is never changed. + pub(crate) fn spawn(&self, command: &mut Command) -> Result { + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, CHILD_DESCRIPTOR_MINIMUM)?; + set_working_directory(command, directory); + command.spawn() + } +} + +impl RepositoryReadPolicy { + #[cfg(test)] + pub(super) const fn read_access(self) -> ReadAccessPolicy { + self.read_access + } + + #[cfg(test)] + pub(super) const fn blocking_io(self) -> BlockingIoPolicy { + self.blocking_io + } + + fn options(self) -> OpenOptions { + let mut options = OpenOptions::new(); + match self.read_access { + ReadAccessPolicy::Enabled => { + options.read(true); + } + } + options.follow(FollowSymlinks::No); + match self.blocking_io { + BlockingIoPolicy::Refuse => { + options.nonblock(true); + } + } + options + } +} + +impl From<&cap_std::fs::Metadata> for DirectoryIdentity { + fn from(metadata: &cap_std::fs::Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + } + } +} + +impl From<&std::fs::Metadata> for RepositoryFileIdentity { + fn from(metadata: &std::fs::Metadata) -> Self { + Self { + device: UnixMetadataExt::dev(metadata), + inode: UnixMetadataExt::ino(metadata), + bytes: metadata.len(), + modified_seconds: UnixMetadataExt::mtime(metadata), + modified_nanoseconds: UnixMetadataExt::mtime_nsec(metadata), + changed_seconds: UnixMetadataExt::ctime(metadata), + changed_nanoseconds: UnixMetadataExt::ctime_nsec(metadata), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/xtask/src/repository_file/exact_copy.rs b/xtask/src/repository_file/exact_copy.rs new file mode 100644 index 0000000..c2c815b --- /dev/null +++ b/xtask/src/repository_file/exact_copy.rs @@ -0,0 +1,48 @@ +//! This module owns bounded exact descriptor-to-file copying. + +use std::fs::File; +use std::io::{self, Write}; +use std::os::unix::fs::FileExt; + +const COPY_BUFFER_BYTES: usize = 16_384; + +/// Copies exactly the admitted byte length from a descriptor at offset zero. +/// +/// The fixed-size buffer bounds memory use. Short sources, offset overflow, +/// unrepresentable read lengths, and destination write failures are refused. +pub(crate) fn copy_exact( + source: &File, + destination: &mut File, + expected: u64, +) -> Result<(), io::Error> { + let mut offset = 0_u64; + let mut buffer = [0_u8; COPY_BUFFER_BYTES]; + while offset < expected { + let remaining = expected + .checked_sub(offset) + .ok_or_else(|| io::Error::other("source offset exceeded its admitted length"))?; + let limit = + usize::try_from(remaining).map_or(buffer.len(), |bytes| bytes.min(buffer.len())); + let chunk = buffer + .get_mut(..limit) + .ok_or_else(|| io::Error::other("read bound exceeded the copy buffer"))?; + let read = source.read_at(chunk, offset)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "source ended before its admitted length", + )); + } + let copied = buffer + .get(..read) + .ok_or_else(|| io::Error::other("write bound exceeded the copy buffer"))?; + destination.write_all(copied)?; + offset = offset + .checked_add( + u64::try_from(read) + .map_err(|_| io::Error::other("source read length is not representable"))?, + ) + .ok_or_else(|| io::Error::other("source offset overflowed"))?; + } + Ok(()) +} diff --git a/xtask/src/repository_file/tests.rs b/xtask/src/repository_file/tests.rs new file mode 100644 index 0000000..da42aec --- /dev/null +++ b/xtask/src/repository_file/tests.rs @@ -0,0 +1,8 @@ +//! This module owns repository descriptor-allocation policy tests. + +use super::CHILD_DESCRIPTOR_MINIMUM; + +#[test] +fn child_directory_descriptors_stay_above_standard_streams() { + assert_eq!(CHILD_DESCRIPTOR_MINIMUM, 3); +} diff --git a/xtask/src/repository_fixture.rs b/xtask/src/repository_fixture.rs new file mode 100644 index 0000000..3559690 --- /dev/null +++ b/xtask/src/repository_fixture.rs @@ -0,0 +1,48 @@ +//! This module owns hermetic Git commands for repository regression fixtures. + +use std::env; +use std::error::Error; +use std::io; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::bounded_process; + +const GIT_FIXTURE_DEADLINE: Duration = Duration::from_mins(2); + +/// Runs one Git fixture command under the repository's bounded process policy. +/// +/// The call blocks for at most two minutes, starts Git in a dedicated process +/// group, discards both output streams, and terminates descendants on timeout +/// or interruption. The child inherits only the admitted executable search +/// path, deterministic locale, and null system and global Git configuration. +/// Process failures retain their typed source; a nonzero Git status reports the +/// attempted arguments without admitting tool output. +pub(crate) fn run_git(root: &Path, arguments: &[&str]) -> Result<(), Box> { + let path = env::var_os("PATH") + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "PATH is unavailable"))?; + let mut command = Command::new("git"); + command + .args(arguments) + .current_dir(root) + .env_clear() + .env("PATH", path) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .env("LC_ALL", "C") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let output = bounded_process::status( + "repository Git fixture", + &mut command, + Some(GIT_FIXTURE_DEADLINE), + )?; + if output.succeeded { + Ok(()) + } else { + Err(format!("git fixture command failed: {arguments:?}").into()) + } +} diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index df78614..4f1ffa4 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -1,40 +1,47 @@ //! This module owns source inventory orchestration and the 500-line law. -mod git_path_inventory; -mod git_path_stream; +mod python_source; mod repository_path; mod source_error; mod source_file; +mod source_inventory; +mod source_kind; -use std::collections::BTreeSet; +use std::collections::BTreeMap; use std::io::{self, BufRead, BufReader}; use std::path::Path; -use git_path_inventory::git_paths; -use git_path_stream::GitPathRecord; +use crate::repository_file::RepositoryRoot; use repository_path::RepositoryPath; -pub(super) use source_error::{GitOutputUnit, SourceStructureError}; -use source_file::{OpenSourceError, SourceRoot}; +pub(super) use source_error::SourceStructureError; +use source_file::{AdmittedSource, FileExecution, SourceFileAdmission, TrackedFileMode}; +#[cfg(test)] +use source_inventory::{ + PRESENT_PATH_ARGUMENTS, select as select_source_inventory, select_source_paths, +}; +use source_inventory::{SourceInventory, collect as source_paths}; +use source_kind::is_extensionless_file; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; -const SOURCE_SUFFIXES: [[u8; 2]; 3] = [*b"py", *b"rs", *b"sh"]; -const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", -]; pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> { let source_root = - SourceRoot::open(repository_root).map_err(|source| SourceStructureError::Inspect { + RepositoryRoot::open(repository_root).map_err(|source| SourceStructureError::Inspect { path: repository_root.to_owned(), source, })?; - let paths = source_paths(repository_root)?; verify_source_root(&source_root, repository_root)?; - let violations = source_violations(&source_root, paths)?; + let process_directory = + source_root + .process_directory() + .map_err(|source| SourceStructureError::Inspect { + path: repository_root.to_owned(), + source, + })?; + let paths = source_paths(&process_directory)?; + verify_source_root(&source_root, repository_root)?; + let violations = inventory_violations(&source_root, paths)?; + verify_source_root(&source_root, repository_root)?; if violations.is_empty() { Ok(()) } else { @@ -46,7 +53,7 @@ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> } fn verify_source_root( - source_root: &SourceRoot, + source_root: &RepositoryRoot, repository_root: &Path, ) -> Result<(), SourceStructureError> { match source_root.is_current_path() { @@ -61,91 +68,89 @@ fn verify_source_root( } } -fn source_paths(repository_root: &Path) -> Result, SourceStructureError> { - let present = git_paths( - repository_root, - &PRESENT_PATH_ARGUMENTS, - "git ls-files present", - )?; - let deleted = git_paths( - repository_root, - &["ls-files", "-z", "--deleted"], - "git ls-files deleted", - )?; - select_source_paths(&present, &deleted) -} - -fn select_source_paths( - present: &BTreeSet, - deleted: &BTreeSet, -) -> Result, SourceStructureError> { - present - .difference(deleted) - .filter(|path| is_source_module(path.as_bytes())) - .map(admit_source_path) - .collect() -} - -fn admit_source_path(path: &GitPathRecord) -> Result { - let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { - SourceStructureError::GitPathEncoding { - operation: "source path admission", - source, +fn inventory_violations( + source_root: &RepositoryRoot, + inventory: SourceInventory, +) -> Result, SourceStructureError> { + let SourceInventory { + modules, + executable_candidates, + tracked_modes, + } = inventory; + let mut violations = Vec::new(); + for relative in executable_candidates { + let tracked_mode = tracked_modes.get(relative.as_path()).copied(); + let source = admit_regular(source_root, relative.as_path(), tracked_mode)?; + if source.execution() == FileExecution::Executable + && source_line_count(source_root, &source)? == SourceLineCount::Exceeded + { + violations.push(relative.as_path().to_owned()); } - })?; - RepositoryPath::admit(text) + } + violations.extend(source_violations_with_modes( + source_root, + modules, + &tracked_modes, + )?); + violations.sort(); + Ok(violations) } +#[cfg(test)] fn source_violations( - source_root: &SourceRoot, + source_root: &RepositoryRoot, + paths: Vec, +) -> Result, SourceStructureError> { + source_violations_with_modes(source_root, paths, &BTreeMap::new()) +} + +fn source_violations_with_modes( + source_root: &RepositoryRoot, paths: Vec, -) -> Result, SourceStructureError> { + tracked_modes: &BTreeMap, +) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { - let lines = source_line_count(source_root, &relative)?; + let tracked_mode = tracked_modes.get(relative.as_path()).copied(); + let source = admit_regular(source_root, relative.as_path(), tracked_mode)?; + if is_extensionless_file(relative.as_str().as_bytes()) + && source.execution() == FileExecution::NonExecutable + { + continue; + } + let lines = source_line_count(source_root, &source)?; if lines == SourceLineCount::Exceeded { - violations.push(relative.as_str().to_owned()); + violations.push(relative.as_path().to_owned()); } } Ok(violations) } -fn source_line_count( - source_root: &SourceRoot, - relative: &RepositoryPath, -) -> Result { - source_line_count_with(source_root, relative, SourceRoot::open_file) +fn admit_regular( + source_root: &RepositoryRoot, + relative: &Path, + tracked_mode: Option, +) -> Result { + match AdmittedSource::admit(source_root, relative, tracked_mode)? { + SourceFileAdmission::Regular(source) => Ok(source), + SourceFileAdmission::NonRegular => Err(SourceStructureError::NonRegular( + source_root.display_path(relative), + )), + } } -fn source_line_count_with( - source_root: &SourceRoot, - relative: &RepositoryPath, - open_source: impl FnOnce(&SourceRoot, &Path) -> Result, +fn source_line_count( + source_root: &RepositoryRoot, + source: &AdmittedSource, ) -> Result { - let path = source_root.display_path(relative.as_path()); - let file = open_source(source_root, relative.as_path()).map_err(|error| match error { - OpenSourceError::Io(source) => SourceStructureError::Inspect { - path: path.clone(), - source, - }, - OpenSourceError::NonRegular => SourceStructureError::NonRegular(path.clone()), + let lines = line_count(BufReader::new(source.file())).map_err(|error| { + SourceStructureError::Inspect { + path: source.path().to_owned(), + source: error, + } })?; - line_count(BufReader::new(file)) - .map_err(|source| SourceStructureError::Inspect { path, source }) -} - -fn is_source_module(path: &[u8]) -> bool { - let Some(file_name) = path.rsplit(|byte| *byte == b'/').next() else { - return false; - }; - let mut components = file_name.rsplitn(2, |byte| *byte == b'.'); - let Some(suffix) = components.next() else { - return false; - }; - let Some(stem) = components.next() else { - return false; - }; - !stem.is_empty() && SOURCE_SUFFIXES.iter().any(|candidate| suffix == candidate) + source.verify_current(source_root)?; + Ok(lines) } const fn exceeds_hard_limit(lines: u64) -> bool { @@ -213,5 +218,11 @@ impl LineCounter { } } +#[cfg(test)] +#[path = "source_structure/executable_candidate_tests.rs"] +mod executable_candidate_tests; +#[cfg(test)] +#[path = "source_structure/pure_rust_tests.rs"] +mod pure_rust_tests; #[cfg(test)] mod tests; diff --git a/xtask/src/source_structure/executable_candidate_tests.rs b/xtask/src/source_structure/executable_candidate_tests.rs new file mode 100644 index 0000000..1e9389c --- /dev/null +++ b/xtask/src/source_structure/executable_candidate_tests.rs @@ -0,0 +1,117 @@ +//! This module owns executable admission outside known source suffixes. + +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use crate::git_inventory::GitPath; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +#[test] +fn non_source_suffix_cannot_hide_executable_python() -> Result<(), Box> { + let directory = TestDirectory::create("hidden-python")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check.txt"); + fs::write(&script, b"#!/usr/bin/python3\nprint('forbidden')\n")?; + make_executable(&script)?; + let present = BTreeSet::from([GitPath::new(b"check.txt".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let result = super::inventory_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) + if path == Path::new("check.txt") + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn non_executable_text_remains_outside_the_python_boundary() +-> Result<(), Box> { + let directory = TestDirectory::create("non-executable-text")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + fs::write( + repository.join("check.txt"), + b"#!/usr/bin/python3\nnot executable\n", + )?; + let present = BTreeSet::from([GitPath::new(b"check.txt".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let violations = super::inventory_violations(&source_root, paths)?; + + assert!(violations.is_empty()); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn non_source_suffix_executable_obeys_the_line_limit() -> Result<(), Box> { + let directory = TestDirectory::create("hidden-source-limit")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check.bash"); + let limit = usize::try_from(super::SOURCE_MODULE_HARD_LIMIT_LINES)?; + fs::write(&script, format!("#!/bin/sh\n{}", ":\n".repeat(limit)))?; + make_executable(&script)?; + let present = BTreeSet::from([GitPath::new(b"check.bash".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let violations = super::inventory_violations(&source_root, paths)?; + + assert_eq!(violations, vec![Path::new("check.bash").to_owned()]); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn non_utf8_executable_candidate_preserves_path_bytes() -> Result<(), Box> { + let path_bytes = b"check-\xff.txt"; + let present = BTreeSet::from([GitPath::new(path_bytes.to_vec())]); + + let inventory = super::select_source_inventory(&present, &BTreeSet::new())?; + + assert_eq!( + inventory + .executable_candidates + .first() + .map(|path| path.as_path().as_os_str().as_bytes()), + Some(path_bytes.as_slice()) + ); + Ok(()) +} + +#[test] +fn every_present_path_is_partitioned_once() -> Result<(), Box> { + let present = BTreeSet::from([ + GitPath::new(b"module.rs".to_vec()), + GitPath::new(b"script.txt".to_vec()), + GitPath::new(b"removed.sh".to_vec()), + ]); + let deleted = BTreeSet::from([GitPath::new(b"removed.sh".to_vec())]); + + let inventory = super::select_source_inventory(&present, &deleted)?; + + assert_eq!(inventory.modules.len(), 1); + assert_eq!(inventory.executable_candidates.len(), 1); + Ok(()) +} + +fn make_executable(path: &Path) -> Result<(), std::io::Error> { + let mut permissions = fs::metadata(path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions) +} diff --git a/xtask/src/source_structure/git_path_inventory.rs b/xtask/src/source_structure/git_path_inventory.rs deleted file mode 100644 index 5855ab4..0000000 --- a/xtask/src/source_structure/git_path_inventory.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! This module owns bounded Git path streaming and diagnostic collection. - -use std::collections::BTreeSet; -use std::io; -use std::path::Path; -use std::process::{Child, ChildStdout, Command, Stdio}; -use std::thread::{self, JoinHandle}; - -use crate::process_output::{BoundedBytes, bounded_bytes}; - -use super::git_path_stream::{GitPathRecord, read_paths}; -use super::{GitOutputUnit, SourceStructureError}; - -const GIT_DIAGNOSTIC_LIMIT_BYTES: usize = 65_536; - -struct GitProcess { - child: Child, - diagnostic_worker: JoinHandle>, - stdout: ChildStdout, -} - -/// Lists repository paths without allowing either child pipe to block. -/// -/// Standard error is drained concurrently before standard output is read. -/// Collection reads standard output before requesting termination, waits for -/// the child before joining the diagnostic reader, and then preserves the -/// established error precedence. -pub(super) fn git_paths( - repository_root: &Path, - arguments: &[&str], - operation: &'static str, -) -> Result, SourceStructureError> { - let process = start_git(repository_root, arguments, operation)?; - let paths = read_paths(process.stdout, operation); - collect_git_result(process.child, process.diagnostic_worker, paths, operation) -} - -fn start_git( - repository_root: &Path, - arguments: &[&str], - operation: &'static str, -) -> Result { - let mut child = Command::new("git") - .args(arguments) - .current_dir(repository_root) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|source| SourceStructureError::RunGit { - operation, - action: "start", - source, - })?; - let Some(stdout) = child.stdout.take() else { - cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::GitPipe { - operation, - stream: "stdout", - }); - }; - let Some(stderr) = child.stderr.take() else { - cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::GitPipe { - operation, - stream: "stderr", - }); - }; - let diagnostic_worker = thread::Builder::new() - .name(String::from("xtask-git-diagnostic")) - .spawn(move || bounded_bytes(stderr, GIT_DIAGNOSTIC_LIMIT_BYTES)); - let diagnostic_worker = match diagnostic_worker { - Ok(worker) => worker, - Err(source) => { - cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::RunGit { - operation, - action: "start the diagnostic reader for", - source, - }); - } - }; - Ok(GitProcess { - child, - diagnostic_worker, - stdout, - }) -} - -fn collect_git_result( - mut child: Child, - diagnostic_worker: JoinHandle>, - paths: Result, SourceStructureError>, - operation: &'static str, -) -> Result, SourceStructureError> { - let stop = if paths.is_err() { - request_stop(&mut child, operation) - } else { - Ok(()) - }; - let status = child.wait().map_err(|source| SourceStructureError::RunGit { - operation, - action: "wait for", - source, - }); - let diagnostic = diagnostic_worker - .join() - .map_err(|_| SourceStructureError::GitWorker { operation })?; - - stop?; - let paths = paths?; - let status = status?; - let diagnostic = diagnostic.map_err(|source| SourceStructureError::RunGit { - operation, - action: "read diagnostics from", - source, - })?; - if diagnostic.exceeded { - return Err(SourceStructureError::GitOutputBound { - operation, - stream: "diagnostic bytes", - maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, - unit: GitOutputUnit::Bytes, - }); - } - if !status.success() { - return Err(git_failure(operation, status.code(), diagnostic.bytes)); - } - Ok(paths) -} - -fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), SourceStructureError> { - child - .kill() - .or_else(|source| { - if source.kind() == io::ErrorKind::InvalidInput { - Ok(()) - } else { - Err(source) - } - }) - .map_err(|source| SourceStructureError::RunGit { - operation, - action: "stop", - source, - }) -} - -fn cleanup_child(child: &mut Child, operation: &'static str) -> Result<(), SourceStructureError> { - let stop = request_stop(child, operation); - let wait = child.wait().map_err(|source| SourceStructureError::RunGit { - operation, - action: "wait for", - source, - }); - stop?; - wait.map(|_| ()) -} - -fn git_failure( - operation: &'static str, - code: Option, - diagnostic: Vec, -) -> SourceStructureError { - match String::from_utf8(diagnostic) { - Ok(stderr) => SourceStructureError::GitFailed { - operation, - code, - stderr, - }, - Err(source) => SourceStructureError::GitDiagnosticEncoding { - operation, - code, - source, - }, - } -} - -#[cfg(test)] -mod tests; diff --git a/xtask/src/source_structure/git_path_inventory/tests.rs b/xtask/src/source_structure/git_path_inventory/tests.rs deleted file mode 100644 index 6233753..0000000 --- a/xtask/src/source_structure/git_path_inventory/tests.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! This module owns adversarial Git diagnostic-bound tests. - -use std::io::Cursor; - -use super::{SourceStructureError, git_failure}; -use crate::process_output::bounded_bytes; - -#[test] -fn git_diagnostic_encoding_failure_retains_exit_status() { - let error = git_failure("test diagnostics", Some(9), vec![u8::MAX]); - assert!(matches!( - error, - SourceStructureError::GitDiagnosticEncoding { - operation: "test diagnostics", - code: Some(9), - .. - } - )); -} - -#[test] -fn git_diagnostics_are_drained_but_only_the_bound_is_retained() { - let result = bounded_bytes(Cursor::new(b"abcdef"), 3); - assert!(matches!( - result, - Ok(ref diagnostic) if diagnostic.bytes == b"abc" && diagnostic.exceeded - )); -} diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs new file mode 100644 index 0000000..16cc1a4 --- /dev/null +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -0,0 +1,189 @@ +//! This module owns the pure-Rust source-admission regression law. + +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; + +use crate::git_inventory::GitPath; +use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; +use crate::test_directory::TestDirectory; + +#[test] +fn python_source_is_refused_by_the_pure_rust_boundary() { + for path in [ + ".py", + "scripts/.PYW", + "scripts/check.py", + "scripts/check.Py", + "scripts/check.PY", + "scripts/check.pyw", + "scripts/check.pYw", + "scripts/check.PYW", + ] { + let present = BTreeSet::from([GitPath::new(path.as_bytes().to_vec())]); + assert!(matches!( + super::select_source_paths(&present, &BTreeSet::new()), + Err(super::SourceStructureError::PythonSource(ref observed)) + if observed == path + )); + } +} + +#[test] +fn extensionless_executable_python_is_refused_by_the_pure_rust_boundary() +-> Result<(), Box> { + let fixture = SourceFixture::create( + "extensionless-python", + "check", + b"#!/usr/bin/env python3\nprint('forbidden')\n", + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; + let result = super::source_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) if path == "check" + )); + drop(source_root); + fixture.close()?; + Ok(()) +} + +#[test] +fn extension_bearing_executable_python_is_refused_by_the_pure_rust_boundary() +-> Result<(), Box> { + let fixture = SourceFixture::create( + "extension-bearing-python", + "check.sh", + b"#!/usr/bin/python3\nprint('forbidden')\n", + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; + let result = super::source_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) if path == "check.sh" + )); + drop(source_root); + fixture.close()?; + Ok(()) +} + +#[test] +fn extensionless_executable_source_obeys_the_line_limit() -> Result<(), Box> +{ + let limit = usize::try_from(super::SOURCE_MODULE_HARD_LIMIT_LINES)?; + let contents = format!("#!/bin/sh\n{}", ":\n".repeat(limit)); + let fixture = SourceFixture::create( + "extensionless-source-limit", + "check", + contents.as_bytes(), + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; + let violations = super::source_violations(&source_root, paths)?; + + assert_eq!(violations, vec![String::from("check")]); + drop(source_root); + fixture.close()?; + Ok(()) +} + +#[test] +fn extensionless_nonexecutable_text_is_not_a_source_module() +-> Result<(), Box> { + let fixture = SourceFixture::create( + "extensionless-text", + "NOTICE", + b"#!/usr/bin/env python3\nnot executable\n", + FixtureMode::NonExecutable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; + assert!(super::source_violations(&source_root, paths)?.is_empty()); + + drop(source_root); + fixture.close()?; + Ok(()) +} + +#[test] +fn tracked_executable_mode_must_match_the_worktree() -> Result<(), Box> { + let directory = TestDirectory::create("tracked-executable-mode")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + run_git(&repository, &["init", "--quiet", "--template="])?; + fs::write( + repository.join("script"), + "#!/usr/bin/env python3\nprint('forbidden')\n", + )?; + run_git(&repository, &["add", "script"])?; + run_git(&repository, &["update-index", "--chmod=+x", "script"])?; + + let result = super::check(&repository); + + assert!(matches!( + result, + Err(super::SourceStructureError::ExecutionModeChanged { + ref path, + tracked: "executable", + worktree: "nonexecutable", + }) if path == &repository.join("script") + )); + directory.close()?; + Ok(()) +} + +#[derive(Clone, Copy)] +enum FixtureMode { + Executable, + NonExecutable, +} + +struct SourceFixture { + directory: TestDirectory, + present: BTreeSet, + repository: PathBuf, +} + +impl SourceFixture { + fn create( + case: &str, + path: &str, + contents: &[u8], + mode: FixtureMode, + ) -> Result> { + let directory = TestDirectory::create(case)?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join(path); + fs::write(&script, contents)?; + let mut permissions = fs::metadata(&script)?.permissions(); + permissions.set_mode(match mode { + FixtureMode::Executable => 0o755, + FixtureMode::NonExecutable => 0o644, + }); + fs::set_permissions(&script, permissions)?; + Ok(Self { + directory, + present: BTreeSet::from([GitPath::new(path.as_bytes().to_vec())]), + repository, + }) + } + + fn close(self) -> Result<(), Box> { + self.directory.close()?; + Ok(()) + } +} diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs new file mode 100644 index 0000000..01d470b --- /dev/null +++ b/xtask/src/source_structure/python_source.rs @@ -0,0 +1,145 @@ +//! This module owns bounded executable Python-shebang admission. + +mod environment; + +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; + +const SHEBANG_SCAN_BYTES: usize = 1_024; + +pub(super) fn executable_uses_python(file: &File) -> Result { + let mut prefix = [0_u8; SHEBANG_SCAN_BYTES]; + let bytes = read_prefix(file, &mut prefix)?; + let admitted = prefix.get(..bytes).ok_or_else(prefix_bounds_error)?; + Ok(is_python_shebang(admitted)) +} + +fn read_prefix(file: &File, prefix: &mut [u8]) -> Result { + let mut filled = 0_usize; + while filled < prefix.len() { + let offset = u64::try_from(filled).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "shebang prefix offset exceeds u64", + ) + })?; + let remaining = prefix.get_mut(filled..).ok_or_else(prefix_bounds_error)?; + let read = match file.read_at(remaining, offset) { + Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + result => result?, + }; + if read == 0 { + break; + } + filled = filled.checked_add(read).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "shebang prefix length overflow") + })?; + } + Ok(filled) +} + +fn prefix_bounds_error() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "shebang prefix bounds are inconsistent", + ) +} + +fn is_python_shebang(prefix: &[u8]) -> bool { + let Some(line) = prefix + .split(|byte| *byte == b'\n') + .next() + .and_then(|line| line.strip_prefix(b"#!")) + else { + return false; + }; + let line = line.trim_ascii_start(); + let mut fields = line.splitn(2, u8::is_ascii_whitespace); + let Some(interpreter) = fields.next().filter(|field| !field.is_empty()) else { + return false; + }; + if is_python_program(interpreter) { + return true; + } + if !program_name(interpreter).eq_ignore_ascii_case(b"env") { + return false; + } + match fields.next().and_then(environment::selected_utility) { + Some(environment::UtilitySelection::Known(utility)) => is_python_program(&utility), + Some(environment::UtilitySelection::Ambiguous) => true, + None => false, + } +} + +fn is_python_program(program: &[u8]) -> bool { + let unquoted = program + .strip_prefix(b"\"") + .or_else(|| program.strip_prefix(b"'")) + .unwrap_or(program); + let name = program_name(unquoted); + starts_with_ignore_ascii_case(name, b"python") || starts_with_ignore_ascii_case(name, b"pypy") +} + +fn program_name(program: &[u8]) -> &[u8] { + program + .rsplit(|byte| *byte == b'/') + .next() + .unwrap_or(program) +} + +fn starts_with_ignore_ascii_case(value: &[u8], prefix: &[u8]) -> bool { + value + .get(..prefix.len()) + .is_some_and(|observed| observed.eq_ignore_ascii_case(prefix)) +} + +#[cfg(test)] +mod tests { + use super::is_python_shebang; + + #[test] + fn direct_and_environment_python_interpreters_are_detected() { + for shebang in [ + b"#!/usr/bin/python3\n".as_slice(), + b"#! /usr/bin/env python3 -I\n", + b"#!/usr/bin/env -S python3 -I\n", + b"#!/usr/bin/env -S \"python3 -I\"\n", + b"#!/usr/bin/env -Spython3 -I\n", + b"#!/usr/bin/env -S/opt/PyPy3 -I\n", + b"#!/usr/bin/env --split-string=python3\n", + b"#!/usr/bin/env --spl=python3 -I\n", + b"#!/usr/bin/env --ignore-e python3\n", + b"#!/opt/PyPy3\n", + ] { + assert!(is_python_shebang(shebang)); + } + } + + #[test] + fn non_python_or_displaced_interpreters_are_not_misclassified() { + for prefix in [ + b"#!/bin/sh\n".as_slice(), + b"#!/usr/bin/env bash\n", + b"#!/usr/bin/env sh -c python3\n", + b"#!/usr/bin/env -S sh -c 'echo python3'\n", + b"#!/usr/bin/env -S \"sh -c 'echo python3'\"\n", + b"python3\n", + b"first line\n#!/usr/bin/python3\n", + ] { + assert!(!is_python_shebang(prefix)); + } + } + + #[test] + fn unresolved_environment_utility_substitution_fails_closed() { + assert!(is_python_shebang( + b"#!/usr/bin/env -S '${UNSET_INTERPRETER}sh'\n" + )); + } + + #[test] + fn combined_environment_options_cannot_hide_python() { + assert!(is_python_shebang(b"#!/usr/bin/env -S-iuFOO python3\n")); + } +} diff --git a/xtask/src/source_structure/python_source/environment.rs b/xtask/src/source_structure/python_source/environment.rs new file mode 100644 index 0000000..e3a2443 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment.rs @@ -0,0 +1,127 @@ +//! This module owns deterministic `env` shebang utility selection. + +mod long_options; +mod short_options; +mod word_split; + +use std::collections::VecDeque; + +use long_options::{LongOptionAction, action as long_option_action}; +use short_options::{ShortOptionAction, action as short_option_action}; +use word_split::split_words; + +/// The selected `env` utility when repository bytes determine it safely. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum UtilitySelection { + /// The exact utility word selected after options and assignments. + Known(Vec), + /// Runtime environment substitution can change the selected utility. + Ambiguous, +} + +enum OptionAction { + Consumed, + End, + Utility, +} + +/// Selects the utility executed by an `env` shebang. +/// +/// Invalid word framing returns absence. A selected word containing unresolved +/// `${VAR}` syntax remains explicitly ambiguous so callers can fail closed. +pub(super) fn selected_utility(arguments: &[u8]) -> Option { + let mut words = VecDeque::from(split_words(arguments)?); + let mut options = true; + let mut split_budget = arguments.len().checked_add(1)?; + while let Some(word) = words.pop_front() { + if options { + match option_action(&word, &mut words, &mut split_budget)? { + OptionAction::Consumed => continue, + OptionAction::End => { + options = false; + continue; + } + OptionAction::Utility => {} + } + } + if word.contains(&b'=') { + options = false; + continue; + } + if word.windows(2).any(|window| window == b"${") { + return Some(UtilitySelection::Ambiguous); + } + return Some(UtilitySelection::Known(word)); + } + None +} + +fn option_action( + word: &[u8], + words: &mut VecDeque>, + split_budget: &mut usize, +) -> Option { + if word == b"--" { + return Some(OptionAction::End); + } + if word == b"-" { + return Some(OptionAction::Consumed); + } + if let Some(action) = long_option_action(word) { + return match action { + LongOptionAction::Consumed => Some(OptionAction::Consumed), + LongOptionAction::TakesNext => { + words.pop_front()?; + Some(OptionAction::Consumed) + } + LongOptionAction::Split(split) => expand_split(split, words, split_budget), + LongOptionAction::Invalid => None, + }; + } + if let Some(split) = short_split_value(word) { + return expand_split(split, words, split_budget); + } + match short_option_action(word) { + Some(ShortOptionAction::Consumed) => Some(OptionAction::Consumed), + Some(ShortOptionAction::TakesNext) => { + words.pop_front()?; + Some(OptionAction::Consumed) + } + Some(ShortOptionAction::Split(split)) => expand_split(split, words, split_budget), + Some(ShortOptionAction::Invalid) => None, + None if word.starts_with(b"-") => None, + None => Some(OptionAction::Utility), + } +} + +fn expand_split( + split: &[u8], + words: &mut VecDeque>, + split_budget: &mut usize, +) -> Option { + *split_budget = split_budget.checked_sub(1)?; + *words = expanded_words(split, std::mem::take(words))?; + Some(OptionAction::Consumed) +} + +fn expanded_words(first: &[u8], remaining: VecDeque>) -> Option>> { + let mut input = first.to_vec(); + for word in remaining { + if !input.is_empty() { + input.push(b' '); + } + input.extend_from_slice(&word); + } + Some(VecDeque::from(split_words(&input)?)) +} + +fn short_split_value(word: &[u8]) -> Option<&[u8]> { + if word == b"-S" { + Some(b"") + } else { + word.strip_prefix(b"-S").filter(|value| !value.is_empty()) + } +} + +#[cfg(test)] +mod tests; diff --git a/xtask/src/source_structure/python_source/environment/long_options.rs b/xtask/src/source_structure/python_source/environment/long_options.rs new file mode 100644 index 0000000..3bc57bb --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/long_options.rs @@ -0,0 +1,100 @@ +//! This module owns GNU `env` long-option abbreviation admission. + +#[derive(Clone, Copy)] +enum ValueMode { + None, + Required, + Optional, + Split, +} + +#[derive(Clone, Copy)] +struct LongOption { + name: &'static [u8], + value: ValueMode, +} + +struct LongOptionWord<'a> { + name: &'a [u8], + attached: Option<&'a [u8]>, +} + +const OPTIONS: [LongOption; 12] = [ + LongOption::new(b"--argv0", ValueMode::Required), + LongOption::new(b"--block-signal", ValueMode::Optional), + LongOption::new(b"--chdir", ValueMode::Required), + LongOption::new(b"--debug", ValueMode::None), + LongOption::new(b"--default-signal", ValueMode::Optional), + LongOption::new(b"--help", ValueMode::None), + LongOption::new(b"--ignore-environment", ValueMode::None), + LongOption::new(b"--ignore-signal", ValueMode::Optional), + LongOption::new(b"--list-signal-handling", ValueMode::None), + LongOption::new(b"--null", ValueMode::None), + LongOption::new(b"--split-string", ValueMode::Split), + LongOption::new(b"--unset", ValueMode::Required), +]; + +/// The effect of one admitted GNU `env` long option. +pub(super) enum LongOptionAction<'a> { + /// The option and any attached or optional value are complete. + Consumed, + /// The option requires the next word as its value. + TakesNext, + /// The option replaces the remaining words with split-string bytes. + Split(&'a [u8]), + /// The option forbids its attached value. + Invalid, +} + +/// Admits an exact or unambiguous abbreviated GNU `env` long option. +pub(super) fn action(word: &[u8]) -> Option> { + let word = split_value(word)?; + if !word.name.starts_with(b"--") { + return None; + } + let option = unique_match(word.name)?; + Some(match (option.value, word.attached) { + (ValueMode::None, None) | (ValueMode::Optional, _) | (ValueMode::Required, Some(_)) => { + LongOptionAction::Consumed + } + (ValueMode::Required, None) => LongOptionAction::TakesNext, + (ValueMode::Split, value) => LongOptionAction::Split(value.unwrap_or_default()), + (ValueMode::None, Some(_)) => LongOptionAction::Invalid, + }) +} + +fn unique_match(name: &[u8]) -> Option { + if let Some(option) = OPTIONS.iter().find(|option| option.name == name) { + return Some(*option); + } + let mut matches = OPTIONS + .iter() + .filter(|option| option.name.starts_with(name)) + .copied(); + let option = matches.next()?; + matches.next().is_none().then_some(option) +} + +fn split_value(word: &[u8]) -> Option> { + match word.iter().position(|byte| *byte == b'=') { + Some(index) => { + let name = word.get(..index)?; + let value_start = index.checked_add(1)?; + let value = word.get(value_start..)?; + Some(LongOptionWord { + name, + attached: Some(value), + }) + } + None => Some(LongOptionWord { + name: word, + attached: None, + }), + } +} + +impl LongOption { + const fn new(name: &'static [u8], value: ValueMode) -> Self { + Self { name, value } + } +} diff --git a/xtask/src/source_structure/python_source/environment/short_options.rs b/xtask/src/source_structure/python_source/environment/short_options.rs new file mode 100644 index 0000000..6fe5492 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/short_options.rs @@ -0,0 +1,30 @@ +//! This module owns deterministic combined short-option decoding for `env`. + +pub(super) enum ShortOptionAction<'a> { + Consumed, + TakesNext, + Split(&'a [u8]), + Invalid, +} + +pub(super) fn action(word: &[u8]) -> Option> { + let mut options = word.strip_prefix(b"-")?; + if options.is_empty() || options.starts_with(b"-") { + return None; + } + loop { + let (option, remaining) = options.split_first()?; + match option { + b'i' | b'v' | b'0' if remaining.is_empty() => { + return Some(ShortOptionAction::Consumed); + } + b'i' | b'v' | b'0' => options = remaining, + b'u' | b'C' | b'a' if remaining.is_empty() => { + return Some(ShortOptionAction::TakesNext); + } + b'u' | b'C' | b'a' => return Some(ShortOptionAction::Consumed), + b'S' => return Some(ShortOptionAction::Split(remaining)), + _ => return Some(ShortOptionAction::Invalid), + } + } +} diff --git a/xtask/src/source_structure/python_source/environment/tests.rs b/xtask/src/source_structure/python_source/environment/tests.rs new file mode 100644 index 0000000..478d956 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/tests.rs @@ -0,0 +1,57 @@ +//! This module owns `env` option and utility-selection laws. + +#[test] +fn options_assignments_and_split_strings_precede_the_utility() { + for arguments in [ + b"-i python3 -I".as_slice(), + b"-iv python3", + b"-u PYTHONHOME python3", + b"NAME=value python3", + b"-S -i python3 -I", + b"-S \"python3 -I\"", + b"-S-iuFOO python3", + b"-iSpython3 -I", + b"--split-string='python3 -I'", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"python3".to_vec())) + ); + } +} + +#[test] +fn arguments_after_the_utility_cannot_replace_it() { + for arguments in [ + b"sh -c python3".as_slice(), + b"-S sh -c 'echo python3'", + b"-S \"sh -c 'echo python3'\"", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"sh".to_vec())) + ); + } +} + +#[test] +fn unresolved_selected_utility_is_ambiguous() { + assert_eq!( + super::selected_utility(b"-S '${UNSET_INTERPRETER}sh'"), + Some(super::UtilitySelection::Ambiguous) + ); +} + +#[test] +fn unambiguous_long_option_abbreviations_preserve_the_utility() { + for arguments in [ + b"--spl=python3 -I".as_slice(), + b"--ignore-e python3", + b"--uns PYTHONHOME python3", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"python3".to_vec())) + ); + } +} diff --git a/xtask/src/source_structure/python_source/environment/word_split.rs b/xtask/src/source_structure/python_source/environment/word_split.rs new file mode 100644 index 0000000..b15c4e0 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/word_split.rs @@ -0,0 +1,70 @@ +//! This module owns bounded shell-word splitting for `env -S`. + +pub(super) fn split_words(input: &[u8]) -> Option>> { + let mut splitter = WordSplitter::default(); + for byte in input { + splitter.observe(*byte); + } + splitter.finish() +} + +#[derive(Default)] +struct WordSplitter { + words: Vec>, + word: Vec, + started: bool, + quote: Quote, + escaped: bool, +} + +impl WordSplitter { + fn observe(&mut self, byte: u8) { + if self.escaped { + self.word.push(byte); + self.escaped = false; + return; + } + match (self.quote, byte) { + (Quote::None, byte) if byte.is_ascii_whitespace() => self.complete_word(), + (Quote::None, b'\'') => self.start_quote(Quote::Single), + (Quote::None, b'"') => self.start_quote(Quote::Double), + (Quote::None | Quote::Double, b'\\') => { + self.escaped = true; + self.started = true; + } + (Quote::Single, b'\'') | (Quote::Double, b'"') => self.quote = Quote::None, + (_, byte) => { + self.word.push(byte); + self.started = true; + } + } + } + + fn complete_word(&mut self) { + if self.started { + self.words.push(std::mem::take(&mut self.word)); + self.started = false; + } + } + + const fn start_quote(&mut self, quote: Quote) { + self.quote = quote; + self.started = true; + } + + fn finish(mut self) -> Option>> { + if self.escaped || self.quote != Quote::None { + return None; + } + self.complete_word(); + Some(self.words) + } +} + +#[derive(Clone, Copy, Default, Eq, PartialEq)] +enum Quote { + #[default] + None, + Single, + Double, +} diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 1200e59..3eba166 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -1,66 +1,43 @@ //! This module owns typed source-structure failures and stable diagnostics. use std::error::Error; -use std::fmt::{self, Write as _}; +use std::fmt; use std::io; use std::path::PathBuf; use std::string::FromUtf8Error; use crate::diagnostic::{escaped_controls, escaped_path}; - -#[derive(Clone, Copy)] -pub(crate) enum GitOutputUnit { - Bytes, - Items, -} +use crate::git_inventory::GitInventoryError; pub(crate) enum SourceStructureError { - DuplicatePath(Vec), - GitFailed { - operation: &'static str, - code: Option, - stderr: String, - }, - GitDiagnosticEncoding { - operation: &'static str, - code: Option, - source: FromUtf8Error, + /// Git's tracked file mode disagrees with the opened worktree object. + ExecutionModeChanged { + /// Ambient display path for the disagreed repository entry. + path: PathBuf, + /// Canonical tracked mode label admitted from the index. + tracked: &'static str, + /// Canonical mode label observed from the opened worktree object. + worktree: &'static str, }, + GitInventory(GitInventoryError), + /// `git ls-files --stage` returned a malformed, unmerged, or duplicate record. + GitIndexRecord, GitPathEncoding { operation: &'static str, source: FromUtf8Error, }, - GitOutputBound { - operation: &'static str, - stream: &'static str, - maximum: usize, - unit: GitOutputUnit, - }, - GitOutputFraming { - operation: &'static str, - }, - GitPipe { - operation: &'static str, - stream: &'static str, - }, - GitWorker { - operation: &'static str, - }, Inspect { path: PathBuf, source: io::Error, }, InvalidPath(String), NonRegular(PathBuf), + PythonSource(PathBuf), RepositoryRootChanged(PathBuf), - RunGit { - operation: &'static str, - action: &'static str, - source: io::Error, - }, + SourceFileChanged(PathBuf), Violations { maximum: u64, - paths: Vec, + paths: Vec, }, } @@ -73,49 +50,24 @@ impl fmt::Debug for SourceStructureError { impl fmt::Display for SourceStructureError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::DuplicatePath(path) => { - formatter.write_str("git returned duplicate path `")?; - escaped_bytes(formatter, path)?; - formatter.write_str("`") - } - Self::GitFailed { - operation, - code, - stderr, - } => git_failed(formatter, operation, *code, stderr), - Self::GitDiagnosticEncoding { - operation, code, .. - } => write!( - formatter, - "`{operation}` failed with code {code:?} and returned non-UTF-8 diagnostics" - ), - Self::GitPathEncoding { operation, .. } => { - write!(formatter, "`{operation}` returned a non-UTF-8 path") - } - Self::GitOutputBound { - operation, - stream, - maximum, - unit, - } => write!( - formatter, - "`{operation}` exceeded the {stream} bound of {maximum} {}", - unit.label() - ), - Self::GitOutputFraming { operation } => { + Self::ExecutionModeChanged { + path, + tracked, + worktree, + } => { + formatter.write_str("repository execution mode differs from the index for `")?; + escaped_path(formatter, path)?; write!( formatter, - "`{operation}` returned a non-NUL-terminated path" + "`: tracked mode is {tracked}, worktree mode is {worktree}" ) } - Self::GitPipe { operation, stream } => { - write!(formatter, "`{operation}` did not provide its {stream} pipe") + Self::GitInventory(error) => write!(formatter, "{error}"), + Self::GitIndexRecord => { + formatter.write_str("git returned an invalid tracked file-mode record") } - Self::GitWorker { operation } => { - write!( - formatter, - "`{operation}` diagnostic reader stopped unexpectedly" - ) + Self::GitPathEncoding { operation, .. } => { + write!(formatter, "`{operation}` returned a non-UTF-8 path") } Self::Inspect { path, .. } => { formatter.write_str("cannot inspect `")?; @@ -132,77 +84,54 @@ impl fmt::Display for SourceStructureError { escaped_path(formatter, path)?; formatter.write_str("`") } + Self::PythonSource(path) => { + formatter.write_str("pure Rust source boundary refuses Python module `")?; + escaped_path(formatter, path)?; + formatter.write_str("`") + } Self::RepositoryRootChanged(path) => { formatter.write_str("repository root changed during source inspection: `")?; escaped_path(formatter, path)?; formatter.write_str("`") } - Self::RunGit { - operation, action, .. - } => write!(formatter, "cannot {action} `{operation}`"), + Self::SourceFileChanged(path) => { + formatter.write_str("repository source changed during inspection: `")?; + escaped_path(formatter, path)?; + formatter.write_str("`") + } Self::Violations { maximum, paths } => violations_display(formatter, *maximum, paths), } } } -impl GitOutputUnit { - const fn label(self) -> &'static str { - match self { - Self::Bytes => "bytes", - Self::Items => "items", - } - } -} - impl Error for SourceStructureError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::GitDiagnosticEncoding { source, .. } | Self::GitPathEncoding { source, .. } => { - Some(source) - } - Self::Inspect { source, .. } | Self::RunGit { source, .. } => Some(source), - Self::DuplicatePath(_) - | Self::GitFailed { .. } - | Self::GitOutputBound { .. } - | Self::GitOutputFraming { .. } - | Self::GitPipe { .. } - | Self::GitWorker { .. } + Self::GitInventory(error) => Some(error), + Self::GitPathEncoding { source, .. } => Some(source), + Self::Inspect { source, .. } => Some(source), + Self::ExecutionModeChanged { .. } + | Self::GitIndexRecord | Self::InvalidPath(_) | Self::NonRegular(_) + | Self::PythonSource(_) | Self::RepositoryRootChanged(_) + | Self::SourceFileChanged(_) | Self::Violations { .. } => None, } } } -fn git_failed( - formatter: &mut fmt::Formatter<'_>, - operation: &str, - code: Option, - stderr: &str, -) -> fmt::Result { - write!(formatter, "`{operation}` failed with code {code:?}: ")?; - escaped_controls(formatter, stderr.trim()) -} - -fn escaped_bytes(formatter: &mut fmt::Formatter<'_>, bytes: &[u8]) -> fmt::Result { - if let Ok(text) = std::str::from_utf8(bytes) { - return escaped_controls(formatter, text); +impl From for SourceStructureError { + fn from(error: GitInventoryError) -> Self { + Self::GitInventory(error) } - for byte in bytes { - if byte.is_ascii_graphic() || *byte == b' ' { - formatter.write_char(char::from(*byte))?; - } else { - write!(formatter, "\\x{byte:02x}")?; - } - } - Ok(()) } fn violations_display( formatter: &mut fmt::Formatter<'_>, maximum: u64, - paths: &[String], + paths: &[PathBuf], ) -> fmt::Result { write!( formatter, @@ -210,7 +139,7 @@ fn violations_display( )?; for path in paths { formatter.write_str("; ")?; - escaped_controls(formatter, path)?; + escaped_path(formatter, path)?; write!(formatter, ": >{maximum}")?; } Ok(()) diff --git a/xtask/src/source_structure/source_file.rs b/xtask/src/source_structure/source_file.rs index bdd94c0..55dfd5e 100644 --- a/xtask/src/source_structure/source_file.rs +++ b/xtask/src/source_structure/source_file.rs @@ -1,125 +1,176 @@ -//! This module owns capability-relative, no-follow source-file admission. -//! -//! The repository source verifier is intentionally supported only on Unix hosts. -//! It binds an opened source root to Unix device and inode identity so that path -//! replacement cannot silently redirect a scan. Supporting another host requires -//! an equivalent stable directory-identity contract before enabling this task. - -use std::fs::File; +//! This module owns one-handle admission of a repository source file. + +use std::fs::{File, Metadata}; use std::io; +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::ambient_authority; -use cap_std::fs::{Dir, OpenOptions}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; + +use super::SourceStructureError; +use super::python_source::executable_uses_python; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum ReadAccessPolicy { - Enabled, +pub(super) enum FileExecution { + Executable, + NonExecutable, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum BlockingIoPolicy { - Refuse, +/// Canonical Git index mode admitted for one tracked source path. +pub(super) enum TrackedFileMode { + /// The index records a symlink or gitlink rather than a regular source. + NonRegular, + /// The index records a regular file with the contained executable state. + Regular(FileExecution), } -#[derive(Clone, Copy)] -pub(super) struct SourceReadPolicy { - read_access: ReadAccessPolicy, - blocking_io: BlockingIoPolicy, +pub(super) enum SourceFileAdmission { + Regular(AdmittedSource), + NonRegular, } -pub(super) const SOURCE_READ_POLICY: SourceReadPolicy = SourceReadPolicy { - read_access: ReadAccessPolicy::Enabled, - blocking_io: BlockingIoPolicy::Refuse, -}; - -pub(super) struct SourceRoot { - directory: Dir, - identity: DirectoryIdentity, +pub(super) struct AdmittedSource { + execution: FileExecution, + file: File, + identity: RepositoryFileIdentity, path: PathBuf, + relative: PathBuf, } -#[derive(Eq, PartialEq)] -struct DirectoryIdentity { - device: u64, - inode: u64, -} - -pub(super) enum OpenSourceError { - Io(io::Error), - NonRegular, -} +impl AdmittedSource { + pub(super) fn admit( + source_root: &RepositoryRoot, + relative: &Path, + tracked_execution: Option, + ) -> Result { + let path = source_root.display_path(relative); + let file = match source_root.open_file(relative) { + Ok(file) => file, + Err(OpenRepositoryFileError::NonRegular) => { + if let Some(TrackedFileMode::Regular(execution)) = tracked_execution { + return Err(mode_changed(&path, execution.label(), "nonregular")); + } + return Ok(SourceFileAdmission::NonRegular); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(SourceStructureError::Inspect { path, source }); + } + }; + let metadata = file + .metadata() + .map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; + let execution = file_execution(&metadata); + admit_execution(&path, tracked_execution, execution)?; + let python = execution == FileExecution::Executable + && executable_uses_python(&file).map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; + let source = Self { + execution, + file, + identity: RepositoryFileIdentity::from(&metadata), + path, + relative: relative.to_owned(), + }; + source.verify_current(source_root)?; + if python { + return Err(SourceStructureError::PythonSource(relative.to_owned())); + } + Ok(SourceFileAdmission::Regular(source)) + } -impl SourceRoot { - pub(super) fn open(path: &Path) -> Result { - let directory = Dir::open_ambient_dir(path, ambient_authority())?; - let identity = DirectoryIdentity::from(&directory.dir_metadata()?); - Ok(Self { - directory, - identity, - path: path.to_owned(), - }) + pub(super) const fn execution(&self) -> FileExecution { + self.execution } - pub(super) fn display_path(&self, relative: &Path) -> PathBuf { - self.path.join(relative) + pub(super) const fn file(&self) -> &File { + &self.file } - pub(super) fn is_current_path(&self) -> Result { - let current = Dir::open_ambient_dir(&self.path, ambient_authority())?; - let identity = DirectoryIdentity::from(¤t.dir_metadata()?); - Ok(self.identity == identity) + pub(super) fn path(&self) -> &Path { + &self.path } - pub(super) fn open_file(&self, relative: &Path) -> Result { - let file = self - .directory - .open_with(relative, &SOURCE_READ_POLICY.options()) - .map_err(OpenSourceError::Io)? - .into_std(); - let metadata = file.metadata().map_err(OpenSourceError::Io)?; - if metadata.is_file() { - Ok(file) + pub(super) fn verify_current( + &self, + source_root: &RepositoryRoot, + ) -> Result<(), SourceStructureError> { + let observed = RepositoryFileIdentity::read(&self.file).map_err(|source| { + SourceStructureError::Inspect { + path: self.path.clone(), + source, + } + })?; + let current = match source_root.open_file(&self.relative) { + Ok(file) => RepositoryFileIdentity::read(&file), + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(SourceStructureError::SourceFileChanged(self.path.clone())); + } + Err(OpenRepositoryFileError::Io(source)) => Err(source), + Err(OpenRepositoryFileError::NonRegular) => { + return Err(SourceStructureError::SourceFileChanged(self.path.clone())); + } + } + .map_err(|source| SourceStructureError::Inspect { + path: self.path.clone(), + source, + })?; + if observed == self.identity && current == self.identity { + Ok(()) } else { - Err(OpenSourceError::NonRegular) + Err(SourceStructureError::SourceFileChanged(self.path.clone())) } } } -impl SourceReadPolicy { - #[cfg(test)] - pub(super) const fn read_access(self) -> ReadAccessPolicy { - self.read_access +fn admit_execution( + path: &Path, + tracked: Option, + worktree: FileExecution, +) -> Result<(), SourceStructureError> { + match tracked { + Some(TrackedFileMode::Regular(tracked)) if tracked != worktree => { + Err(mode_changed(path, tracked.label(), worktree.label())) + } + Some(TrackedFileMode::NonRegular) => { + Err(mode_changed(path, "nonregular", worktree.label())) + } + Some(TrackedFileMode::Regular(_)) | None => Ok(()), } +} - #[cfg(test)] - pub(super) const fn blocking_io(self) -> BlockingIoPolicy { - self.blocking_io +fn mode_changed( + path: &Path, + tracked: &'static str, + worktree: &'static str, +) -> SourceStructureError { + SourceStructureError::ExecutionModeChanged { + path: path.to_owned(), + tracked, + worktree, } +} - fn options(self) -> OpenOptions { - let mut options = OpenOptions::new(); - match self.read_access { - ReadAccessPolicy::Enabled => { - options.read(true); - } +impl FileExecution { + const fn label(self) -> &'static str { + match self { + Self::Executable => "executable", + Self::NonExecutable => "nonexecutable", } - options.follow(FollowSymlinks::No); - match self.blocking_io { - BlockingIoPolicy::Refuse => { - options.nonblock(true); - } - } - options } } -impl From<&cap_std::fs::Metadata> for DirectoryIdentity { - fn from(metadata: &cap_std::fs::Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } +fn file_execution(metadata: &Metadata) -> FileExecution { + if metadata.permissions().mode() & 0o111 == 0 { + FileExecution::NonExecutable + } else { + FileExecution::Executable } } diff --git a/xtask/src/source_structure/source_inventory.rs b/xtask/src/source_structure/source_inventory.rs new file mode 100644 index 0000000..9105b5b --- /dev/null +++ b/xtask/src/source_structure/source_inventory.rs @@ -0,0 +1,169 @@ +//! This module owns deterministic source and executable-candidate inventory. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ffi::OsString; +use std::os::unix::ffi::OsStringExt; +use std::path::{Component, Path, PathBuf}; + +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::RepositoryProcessDirectory; + +use super::repository_path::RepositoryPath; +use super::source_error::SourceStructureError; +use super::source_file::{FileExecution, TrackedFileMode}; +use super::source_kind::{is_python_module, is_source_candidate}; + +pub(super) const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", +]; + +pub(super) struct SourceInventory { + pub(super) modules: Vec, + pub(super) executable_candidates: Vec, + pub(super) tracked_modes: BTreeMap, +} + +/// A repository-relative path admitted for executable inspection. +pub(super) struct InspectionPath(PathBuf); + +impl InspectionPath { + /// Returns the validated platform path. + pub(super) fn as_path(&self) -> &Path { + &self.0 + } +} + +pub(super) fn collect( + process_directory: &RepositoryProcessDirectory, +) -> Result { + let present = paths_with(&PRESENT_PATH_ARGUMENTS, "git ls-files present", |command| { + process_directory.spawn(command) + })?; + let deleted = paths_with( + &["ls-files", "-z", "--deleted"], + "git ls-files deleted", + |command| process_directory.spawn(command), + )?; + let tracked_modes = tracked_modes(process_directory)?; + let mut inventory = select(&present, &deleted)?; + inventory.tracked_modes = tracked_modes; + Ok(inventory) +} + +pub(super) fn select( + present: &BTreeSet, + deleted: &BTreeSet, +) -> Result { + let mut modules = Vec::new(); + let mut executable_candidates = Vec::new(); + for path in present.difference(deleted) { + if is_source_candidate(path.as_bytes()) { + modules.push(admit_source_path(path)?); + } else { + executable_candidates.push(admit_inspection_path(path)?); + } + } + Ok(SourceInventory { + modules, + executable_candidates, + tracked_modes: BTreeMap::new(), + }) +} + +fn tracked_modes( + process_directory: &RepositoryProcessDirectory, +) -> Result, SourceStructureError> { + let records = paths_with( + &["ls-files", "-z", "--cached", "--stage"], + "git ls-files tracked modes", + |command| process_directory.spawn(command), + )?; + let mut modes = BTreeMap::new(); + for record in records { + let (path, mode) = parse_tracked_mode(&record)?; + if modes.insert(path, mode).is_some() { + return Err(SourceStructureError::GitIndexRecord); + } + } + Ok(modes) +} + +fn parse_tracked_mode( + record: &GitPath, +) -> Result<(PathBuf, TrackedFileMode), SourceStructureError> { + let bytes = record.as_bytes(); + let tab = bytes + .iter() + .position(|byte| *byte == b'\t') + .ok_or(SourceStructureError::GitIndexRecord)?; + let path_start = tab + .checked_add(1) + .ok_or(SourceStructureError::GitIndexRecord)?; + let header = bytes + .get(..tab) + .ok_or(SourceStructureError::GitIndexRecord)?; + let path = bytes + .get(path_start..) + .filter(|path| !path.is_empty()) + .ok_or(SourceStructureError::GitIndexRecord)?; + let mut fields = header.split(|byte| *byte == b' '); + let mode = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + let object = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + let stage = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + if object.is_empty() || stage != b"0" || fields.next().is_some() { + return Err(SourceStructureError::GitIndexRecord); + } + let mode = match mode { + b"100644" => TrackedFileMode::Regular(FileExecution::NonExecutable), + b"100755" => TrackedFileMode::Regular(FileExecution::Executable), + b"120000" | b"160000" => TrackedFileMode::NonRegular, + _ => return Err(SourceStructureError::GitIndexRecord), + }; + let path = admit_inspection_path(&GitPath::new(path.to_vec()))?; + Ok((path.0, mode)) +} + +#[cfg(test)] +pub(super) fn select_source_paths( + present: &BTreeSet, + deleted: &BTreeSet, +) -> Result, SourceStructureError> { + select(present, deleted).map(|inventory| inventory.modules) +} + +fn admit_source_path(path: &GitPath) -> Result { + let python = is_python_module(path.as_bytes()); + let text = path_text(path, "source path admission")?; + let relative = RepositoryPath::admit(text)?; + if python { + Err(SourceStructureError::PythonSource( + relative.as_path().to_owned(), + )) + } else { + Ok(relative) + } +} + +fn admit_inspection_path(path: &GitPath) -> Result { + let relative = PathBuf::from(OsString::from_vec(path.as_bytes().to_vec())); + if relative + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + Ok(InspectionPath(relative)) + } else { + Err(SourceStructureError::InvalidPath(path_text( + path, + "executable path admission", + )?)) + } +} + +fn path_text(path: &GitPath, operation: &'static str) -> Result { + String::from_utf8(path.as_bytes().to_vec()) + .map_err(|source| SourceStructureError::GitPathEncoding { operation, source }) +} diff --git a/xtask/src/source_structure/source_kind.rs b/xtask/src/source_structure/source_kind.rs new file mode 100644 index 0000000..2d369ff --- /dev/null +++ b/xtask/src/source_structure/source_kind.rs @@ -0,0 +1,45 @@ +//! This module owns repository source-module classification. + +pub(super) fn is_source_module(path: &[u8]) -> bool { + if is_python_module(path) { + return true; + } + let Some(suffix) = source_suffix(path) else { + return false; + }; + suffix == b"rs" || suffix == b"sh" +} + +pub(super) fn is_python_module(path: &[u8]) -> bool { + repository_file_name(path).is_some_and(|file_name| { + file_name.strip_prefix(b".").is_some_and(is_python_suffix) + || source_suffix(path).is_some_and(is_python_suffix) + }) +} + +pub(super) fn is_source_candidate(path: &[u8]) -> bool { + is_source_module(path) || is_extensionless_file(path) +} + +pub(super) fn is_extensionless_file(path: &[u8]) -> bool { + let Some(file_name) = repository_file_name(path) else { + return false; + }; + !file_name.is_empty() && !file_name.contains(&b'.') +} + +const fn is_python_suffix(suffix: &[u8]) -> bool { + suffix.eq_ignore_ascii_case(b"py") || suffix.eq_ignore_ascii_case(b"pyw") +} + +fn source_suffix(path: &[u8]) -> Option<&[u8]> { + let file_name = repository_file_name(path)?; + let mut components = file_name.rsplitn(2, |byte| *byte == b'.'); + let suffix = components.next()?; + let stem = components.next()?; + (!stem.is_empty()).then_some(suffix) +} + +fn repository_file_name(path: &[u8]) -> Option<&[u8]> { + path.rsplit(|byte| *byte == b'/').next() +} diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index 1987083..8cf5e20 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -1,9 +1,9 @@ //! This module owns source-line, selection, and replacement-race tests. -use super::{ - PRESENT_PATH_ARGUMENTS, SourceLineCount, exceeds_hard_limit, is_source_module, line_count, -}; -use crate::test_directory::TestDirectory; +mod replacement; + +use super::source_kind::is_source_module; +use super::{PRESENT_PATH_ARGUMENTS, SourceLineCount, exceeds_hard_limit, line_count}; use std::io::{self, BufReader, Cursor, Read}; #[test] @@ -47,7 +47,7 @@ fn early_source_refusal_does_not_claim_an_exact_line_count() { let diagnostic = line_count(BufReader::new(reader)).map(|observed| match observed { SourceLineCount::Exceeded => super::SourceStructureError::Violations { maximum: 500, - paths: vec![String::from("src/large.rs")], + paths: vec![std::path::PathBuf::from("src/large.rs")], } .to_string(), SourceLineCount::Within(lines) => format!("unexpected exact count: {lines}"), @@ -63,10 +63,13 @@ fn early_source_refusal_does_not_claim_an_exact_line_count() { #[test] fn source_structure_diagnostics_are_stable() { - let framing = super::SourceStructureError::GitOutputFraming { + use crate::git_inventory::{GitInventoryError, GitOutputUnit}; + + let framing = super::SourceStructureError::GitInventory(GitInventoryError::OutputFraming { operation: "git inventory", - }; + }); let non_regular = super::SourceStructureError::NonRegular("src/link.rs".into()); + let changed = super::SourceStructureError::SourceFileChanged("src/replaced.rs".into()); assert_eq!( framing.to_string(), "`git inventory` returned a non-NUL-terminated path" @@ -75,26 +78,30 @@ fn source_structure_diagnostics_are_stable() { non_regular.to_string(), "repository source module is not a regular file: `src/link.rs`" ); + assert_eq!( + changed.to_string(), + "repository source changed during inspection: `src/replaced.rs`" + ); let violations = super::SourceStructureError::Violations { maximum: 7, - paths: vec![String::from("src/large.rs")], + paths: vec![std::path::PathBuf::from("src/large.rs")], }; assert_eq!( violations.to_string(), "repository source modules exceed the 7-line hard maximum; src/large.rs: >7" ); - let byte_bound = super::SourceStructureError::GitOutputBound { + let byte_bound = super::SourceStructureError::GitInventory(GitInventoryError::OutputBound { operation: "git inventory", stream: "path bytes", maximum: 4_096, - unit: super::GitOutputUnit::Bytes, - }; - let item_bound = super::SourceStructureError::GitOutputBound { + unit: GitOutputUnit::Bytes, + }); + let item_bound = super::SourceStructureError::GitInventory(GitInventoryError::OutputBound { operation: "git inventory", stream: "path count", maximum: 100_000, - unit: super::GitOutputUnit::Items, - }; + unit: GitOutputUnit::Items, + }); assert_eq!( byte_bound.to_string(), "`git inventory` exceeded the path bytes bound of 4096 bytes" @@ -107,11 +114,13 @@ fn source_structure_diagnostics_are_stable() { #[test] fn git_diagnostics_cannot_inject_terminal_control_lines() { - let error = super::SourceStructureError::GitFailed { + use crate::git_inventory::GitInventoryError; + + let error = super::SourceStructureError::GitInventory(GitInventoryError::Failed { operation: "git inventory", code: Some(9), stderr: String::from("first\nError: forged\rrewrite\u{1b}[31m"), - }; + }); let diagnostic = error.to_string(); assert_eq!( diagnostic, @@ -133,125 +142,16 @@ fn git_diagnostics_cannot_inject_terminal_control_lines() { #[test] fn source_read_policy_enables_reads_and_refuses_blocking_io() { - use super::source_file::{BlockingIoPolicy, ReadAccessPolicy, SOURCE_READ_POLICY}; - - assert_eq!(SOURCE_READ_POLICY.read_access(), ReadAccessPolicy::Enabled); - assert_eq!(SOURCE_READ_POLICY.blocking_io(), BlockingIoPolicy::Refuse); -} - -#[cfg(unix)] -#[test] -fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { - use std::fs; - - use super::repository_path::RepositoryPath; - use super::source_file::SourceRoot; - use super::source_line_count; - - let directory = TestDirectory::create("source-root")?; - let root = directory.path().join("repository"); - let retained_root = directory.path().join("retained"); - fs::create_dir(&root)?; - fs::write(root.join("source.rs"), "safe\n")?; - let source_root = SourceRoot::open(&root)?; - let relative = RepositoryPath::admit(String::from("source.rs"))?; - - fs::rename(&root, &retained_root)?; - fs::create_dir(&root)?; - fs::write(root.join("source.rs"), "replacement\n".repeat(501))?; - - let line_count = source_line_count(&source_root, &relative)?; - assert_eq!(line_count, SourceLineCount::Within(1)); - drop(source_root); - directory.close()?; - Ok(()) -} - -#[cfg(unix)] -#[test] -fn source_scan_detects_a_replaced_repository_root() -> Result<(), Box> { - use std::fs; + use crate::repository_file::{BlockingIoPolicy, REPOSITORY_READ_POLICY, ReadAccessPolicy}; - use super::source_file::SourceRoot; - - let directory = TestDirectory::create("source-identity")?; - let root = directory.path().join("repository"); - let retained_root = directory.path().join("retained"); - fs::create_dir(&root)?; - let source_root = SourceRoot::open(&root)?; - - fs::rename(&root, &retained_root)?; - fs::create_dir(&root)?; - let result = super::verify_source_root(&source_root, &root); - assert!(matches!( - result, - Err(super::SourceStructureError::RepositoryRootChanged(ref path)) if path == &root - )); - drop(source_root); - directory.close()?; - Ok(()) -} - -#[cfg(unix)] -#[test] -fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructureError> { - use std::fs; - use std::os::unix::fs::symlink; - - use super::repository_path::RepositoryPath; - use super::source_file::SourceRoot; - use super::source_line_count_with; - - let directory = TestDirectory::create("source-replacement").map_err(|source| { - super::SourceStructureError::Inspect { - path: "scoped test directory".into(), - source, - } - })?; - let root = directory.path().join("repository"); - fs::create_dir(&root).map_err(|source| super::SourceStructureError::Inspect { - path: root.clone(), - source, - })?; - let source_path = root.join("source.rs"); - let retained_path = root.join("retained.rs"); - let target_path = root.join("target.rs"); - fs::write(&source_path, "safe\n").map_err(|source| super::SourceStructureError::Inspect { - path: source_path.clone(), - source, - })?; - fs::write(&target_path, "outside\n".repeat(501)).map_err(|source| { - super::SourceStructureError::Inspect { - path: target_path.clone(), - source, - } - })?; - let source_root = - SourceRoot::open(&root).map_err(|source| super::SourceStructureError::Inspect { - path: root.clone(), - source, - })?; - let relative = RepositoryPath::admit(String::from("source.rs"))?; - - let result = source_line_count_with(&source_root, &relative, |source_root, relative| { - let admitted = source_root.display_path(relative); - fs::rename(&admitted, &retained_path).map_err(super::OpenSourceError::Io)?; - symlink(&target_path, &admitted).map_err(super::OpenSourceError::Io)?; - source_root.open_file(relative) - }); - let refused = matches!( - result, - Err(super::SourceStructureError::Inspect { - ref path, - source: _, - }) if path == &source_path + assert_eq!( + REPOSITORY_READ_POLICY.read_access(), + ReadAccessPolicy::Enabled + ); + assert_eq!( + REPOSITORY_READ_POLICY.blocking_io(), + BlockingIoPolicy::Refuse ); - assert!(refused); - drop(source_root); - directory - .close() - .map_err(|source| super::SourceStructureError::Inspect { path: root, source })?; - Ok(()) } #[test] @@ -263,7 +163,10 @@ fn source_module_limit_accepts_five_hundred_and_refuses_five_hundred_one() { #[test] fn source_module_classification_is_explicit() { assert!(is_source_module(b"src/lib.rs")); + assert!(is_source_module(b".py")); + assert!(is_source_module(b"scripts/.PYW")); assert!(is_source_module(b"scripts/check.py")); + assert!(is_source_module(b"scripts/check.pyw")); assert!(is_source_module(b"scripts/check.sh")); assert!(!is_source_module(b"README.md")); assert!(!is_source_module(b"src/lib.RS")); @@ -285,11 +188,11 @@ fn source_paths_refuse_host_dependent_spellings() { fn source_selection_refuses_an_unsafe_source_record() { use std::collections::BTreeSet; - use super::git_path_stream::GitPathRecord; + use crate::git_inventory::GitPath; let present = BTreeSet::from([ - GitPathRecord::new(b"../escape.rs".to_vec()), - GitPathRecord::new(b"notes/x:y".to_vec()), + GitPath::new(b"../escape.rs".to_vec()), + GitPath::new(b"notes/x:y".to_vec()), ]); let result = super::select_source_paths(&present, &BTreeSet::new()); assert!(matches!( @@ -302,9 +205,9 @@ fn source_selection_refuses_an_unsafe_source_record() { fn source_selection_refuses_a_non_utf8_source_record() { use std::collections::BTreeSet; - use super::git_path_stream::GitPathRecord; + use crate::git_inventory::GitPath; - let present = BTreeSet::from([GitPathRecord::new(b"bad\xff.rs".to_vec())]); + let present = BTreeSet::from([GitPath::new(b"bad\xff.rs".to_vec())]); let result = super::select_source_paths(&present, &BTreeSet::new()); assert!(matches!( result, diff --git a/xtask/src/source_structure/tests/replacement.rs b/xtask/src/source_structure/tests/replacement.rs new file mode 100644 index 0000000..2817827 --- /dev/null +++ b/xtask/src/source_structure/tests/replacement.rs @@ -0,0 +1,177 @@ +//! This module owns source-root and source-file replacement-race tests. + +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::os::unix::fs::{PermissionsExt, symlink}; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::super::repository_path::RepositoryPath; +use super::super::source_file::{AdmittedSource, FileExecution, SourceFileAdmission}; +use super::super::source_inventory; +use super::super::{SourceLineCount, inventory_violations, source_line_count, verify_source_root}; +use crate::git_inventory::GitPath; + +#[test] +fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { + let directory = TestDirectory::create("source-root")?; + let root = directory.path().join("repository"); + let retained_root = directory.path().join("retained"); + fs::create_dir(&root)?; + fs::write(root.join("source.rs"), "safe\n")?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + + fs::rename(&root, &retained_root)?; + fs::create_dir(&root)?; + fs::write(root.join("source.rs"), "replacement\n".repeat(501))?; + + let source = regular_source(AdmittedSource::admit( + &source_root, + relative.as_path(), + None, + )?)?; + assert_eq!( + source_line_count(&source_root, &source)?, + SourceLineCount::Within(1) + ); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_keeps_one_admitted_file_identity() -> Result<(), Box> { + let directory = TestDirectory::create("source-file-identity")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source"); + let retained_path = root.join("retained"); + fs::write(&source_path, "#!/bin/sh\nsafe\n")?; + fs::set_permissions(&source_path, fs::Permissions::from_mode(0o755))?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source"))?; + + let source = regular_source(AdmittedSource::admit( + &source_root, + relative.as_path(), + None, + )?)?; + assert_eq!(source.execution(), FileExecution::Executable); + fs::rename(&source_path, &retained_path)?; + fs::write(&source_path, "replacement\n".repeat(501))?; + + let result = source_line_count(&source_root, &source); + assert!(matches!( + result, + Err(super::super::SourceStructureError::SourceFileChanged(ref path)) + if path == &source_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_refuses_in_place_mutation() -> Result<(), Box> { + let directory = TestDirectory::create("source-file-mutation")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source.rs"); + fs::write(&source_path, "safe\n")?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + let source = regular_source(AdmittedSource::admit( + &source_root, + relative.as_path(), + None, + )?)?; + + fs::write(&source_path, "mutated\n".repeat(501))?; + + let result = source_line_count(&source_root, &source); + assert!(matches!( + result, + Err(super::super::SourceStructureError::SourceFileChanged(ref path)) + if path == &source_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_detects_a_replaced_repository_root() -> Result<(), Box> { + let directory = TestDirectory::create("source-identity")?; + let root = directory.path().join("repository"); + let retained_root = directory.path().join("retained"); + fs::create_dir(&root)?; + let source_root = RepositoryRoot::open(&root)?; + + fs::rename(&root, &retained_root)?; + fs::create_dir(&root)?; + let result = verify_source_root(&source_root, &root); + assert!(matches!( + result, + Err(super::super::SourceStructureError::RepositoryRootChanged(ref path)) if path == &root + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_open_refuses_replacement_symlink() -> Result<(), Box> { + let directory = TestDirectory::create("source-replacement")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source.rs"); + let retained_path = root.join("retained.rs"); + let target_path = root.join("target.rs"); + fs::write(&source_path, "safe\n")?; + fs::write(&target_path, "outside\n".repeat(501))?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + + fs::rename(&source_path, &retained_path)?; + symlink(&target_path, &source_path)?; + let admission = AdmittedSource::admit(&source_root, relative.as_path(), None)?; + assert!(matches!(admission, SourceFileAdmission::NonRegular)); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn nonregular_executable_candidate_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("nonregular-executable-candidate")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let candidate_path = root.join("script.bin"); + let target_path = root.join("target"); + fs::write(&target_path, "#!/bin/sh\n")?; + symlink(&target_path, &candidate_path)?; + let source_root = RepositoryRoot::open(&root)?; + let present = BTreeSet::from([GitPath::new(b"script.bin".to_vec())]); + let inventory = source_inventory::select(&present, &BTreeSet::new())?; + + let result = inventory_violations(&source_root, inventory); + + assert!(matches!( + result, + Err(super::super::SourceStructureError::NonRegular(ref path)) + if path == &candidate_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +fn regular_source(admission: SourceFileAdmission) -> Result { + match admission { + SourceFileAdmission::Regular(source) => Ok(source), + SourceFileAdmission::NonRegular => Err(io::Error::other("expected a regular source file")), + } +} diff --git a/xtask/src/task_error.rs b/xtask/src/task_error.rs index e839669..2df59ae 100644 --- a/xtask/src/task_error.rs +++ b/xtask/src/task_error.rs @@ -5,6 +5,7 @@ use std::fmt; use crate::benchmark_baseline::BenchmarkBaselineError; use crate::diagnostic::escaped_controls; +use crate::documentation_integrity::DocumentationError; use crate::fuzz_campaign::FuzzCampaignError; use crate::fuzz_seed_corpus::FuzzSeedError; use crate::golden_file_worldline::GoldenError; @@ -14,6 +15,7 @@ use crate::source_structure::SourceStructureError; pub(super) enum TaskError { BenchmarkBaseline(BenchmarkBaselineError), Conformance(ConformanceError), + Documentation(DocumentationError), FuzzCampaign(FuzzCampaignError), FuzzSeed(FuzzSeedError), Golden(GoldenError), @@ -37,6 +39,7 @@ impl fmt::Display for TaskError { match self { Self::BenchmarkBaseline(error) => write!(formatter, "{error}"), Self::Conformance(error) => write!(formatter, "{error}"), + Self::Documentation(error) => write!(formatter, "{error}"), Self::FuzzCampaign(error) => write!(formatter, "{error}"), Self::FuzzSeed(error) => write!(formatter, "{error}"), Self::Golden(error) => write!(formatter, "{error}"), @@ -62,7 +65,9 @@ impl fmt::Display for TaskError { "usage: cargo xtask \ ", ), } @@ -74,6 +79,7 @@ impl Error for TaskError { match self { Self::BenchmarkBaseline(error) => Some(error), Self::Conformance(error) => Some(error), + Self::Documentation(error) => Some(error), Self::FuzzCampaign(error) => Some(error), Self::FuzzSeed(error) => Some(error), Self::Golden(error) => Some(error), @@ -100,6 +106,12 @@ impl From for TaskError { } } +impl From for TaskError { + fn from(error: DocumentationError) -> Self { + Self::Documentation(error) + } +} + impl From for TaskError { fn from(error: FuzzCampaignError) -> Self { Self::FuzzCampaign(error) diff --git a/xtask/src/test_directory.rs b/xtask/src/test_directory.rs index 5d5f73d..d0bdf05 100644 --- a/xtask/src/test_directory.rs +++ b/xtask/src/test_directory.rs @@ -8,12 +8,14 @@ use std::sync::atomic::{AtomicU64, Ordering}; const CREATION_ATTEMPTS: u16 = 1_024; static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); +/// A collision-resistant temporary directory removed at the explicit boundary. pub(crate) struct TestDirectory { path: PathBuf, active: bool, } impl TestDirectory { + /// Creates one unique directory beneath the platform temporary directory. pub(crate) fn create(label: &str) -> Result { for _ in 0_u16..CREATION_ATTEMPTS { let sequence = next_sequence()?; @@ -31,10 +33,12 @@ impl TestDirectory { )) } + /// Returns the exact directory path owned by this scope. pub(crate) fn path(&self) -> &Path { &self.path } + /// Removes the exact owned directory tree and disables drop fallback. pub(crate) fn close(mut self) -> Result<(), io::Error> { let removal = match fs::remove_dir_all(&self.path) { Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index b52bc3a..4be148d 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -18,15 +18,6 @@ fn repository_tasks_require_the_committed_dependency_graph() { assert!(CARGO_CONFIGURATION.contains("xtask = \"run --quiet --locked --package xtask --\"")); } -#[test] -fn successful_verification_is_silent() -> Result<(), io::Error> { - let output = invoke(&["verify"])?; - assert!(output.status.success()); - assert!(output.stdout.is_empty()); - assert!(output.stderr.is_empty()); - Ok(()) -} - #[test] fn chunk_identity_conformance_is_repository_owned_and_silent() -> Result<(), io::Error> { let output = invoke(&["chunk-id-conformance-check"])?; @@ -70,12 +61,14 @@ fn fuzz_description_emits_the_admitted_smoke_policy() -> Result<(), io::Error> { assert_eq!( output.stdout, b"CARGO_FUZZ_VERSION: 0.13.2\n\ + FUZZ_BUILD_TIMEOUT_SECONDS: 600\n\ FUZZ_CMIN_SECONDS_PER_TARGET: 120\n\ FUZZ_CORPUS_MAX_BYTES: 536870912\n\ FUZZ_CORPUS_MAX_FILES: 20000\n\ FUZZ_CORPUS_RETENTION_DAYS: 14\n\ FUZZ_INPUT_TIMEOUT_SECONDS: 5\n\ FUZZ_MAX_INPUT_BYTES: 1048576\n\ + FUZZ_PROCESS_GRACE_SECONDS: 60\n\ FUZZ_RSS_LIMIT_MB: 1024\n\ FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS: 30\n\ FUZZ_SECONDS_PER_TARGET: 15\n\ @@ -96,7 +89,9 @@ fn missing_command_returns_the_versioned_usage_contract() -> Result<(), io::Erro b"Error: usage: cargo xtask \ \n" ); Ok(()) diff --git a/xtask/tests/cli_contract/documentation_tools.rs b/xtask/tests/cli_contract/documentation_tools.rs new file mode 100644 index 0000000..7cd8695 --- /dev/null +++ b/xtask/tests/cli_contract/documentation_tools.rs @@ -0,0 +1,132 @@ +//! This module owns hermetic fake documentation tools for CLI verification. + +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module owns this private fixture" +)] + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static SEQUENCE: AtomicU64 = AtomicU64::new(0); + +pub(crate) struct DocumentationTools { + root: Option, +} + +impl DocumentationTools { + pub(crate) fn create() -> Result { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or_else(|| io::Error::other("xtask manifest has no repository parent"))?; + let parent = repository_root.join("target/cli-contract-tools"); + fs::create_dir_all(&parent)?; + let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed); + let root = parent.join(format!("{}-{sequence}", std::process::id())); + fs::create_dir(&root)?; + let tools = Self { root: Some(root) }; + fs::create_dir(tools.bin()?)?; + fs::create_dir(tools.markers()?)?; + tools.install( + "markdownlint-cli2", + "--version", + "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + )?; + tools.install("lychee", "--version", "lychee 0.21.0")?; + tools.install("actionlint", "-version", "1.7.12")?; + Ok(tools) + } + + pub(crate) fn invoke(&self, arguments: &[&str]) -> Result { + Command::new(env!("CARGO_BIN_EXE_xtask")) + .args(arguments) + .env("PATH", self.path_environment()?) + .output() + } + + pub(crate) fn require_every_tool(&self) -> Result<(), io::Error> { + for program in ["markdownlint-cli2", "lychee", "actionlint"] { + let marker = self.markers()?.join(program); + if !marker.is_file() { + return Err(io::Error::other(format!( + "documentation tool was not invoked: {program}" + ))); + } + } + Ok(()) + } + + pub(crate) fn close(mut self) -> Result<(), io::Error> { + let root = self + .root + .take() + .ok_or_else(|| io::Error::other("documentation tool directory is already closed"))?; + fs::remove_dir_all(root) + } + + fn install( + &self, + program: &str, + version_argument: &str, + version: &str, + ) -> Result<(), io::Error> { + let marker = shell_word(&self.markers()?.join(program))?; + let script = format!( + "#!/bin/sh\n\ + : > {marker}\n\ + for argument in \"$@\"; do\n\ + \x20 if [ \"$argument\" = \"{version_argument}\" ]; then\n\ + \x20 printf '%s\\n' '{version}'\n\ + \x20 exit 0\n\ + \x20 fi\n\ + done\n\ + exit 0\n" + ); + let path = self.bin()?.join(program); + fs::write(&path, script)?; + let mut permissions = fs::metadata(&path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions) + } + + fn path_environment(&self) -> Result { + let existing = env::var_os("PATH").unwrap_or_default(); + env::join_paths(std::iter::once(self.bin()?).chain(env::split_paths(&existing))) + .map_err(io::Error::other) + } + + fn bin(&self) -> Result { + Ok(self.root()?.join("bin")) + } + + fn markers(&self) -> Result { + Ok(self.root()?.join("markers")) + } + + fn root(&self) -> Result<&Path, io::Error> { + self.root + .as_deref() + .ok_or_else(|| io::Error::other("documentation tool directory is closed")) + } +} + +fn shell_word(path: &Path) -> Result { + let text = path + .to_str() + .ok_or_else(|| io::Error::other("documentation tool path is not UTF-8"))?; + Ok(format!("'{}'", text.replace('\'', "'\\''"))) +} + +impl Drop for DocumentationTools { + fn drop(&mut self) { + if let Some(root) = self.root.take() { + drop(fs::remove_dir_all(root)); + } + } +} diff --git a/xtask/tests/documentation_cli_contract.rs b/xtask/tests/documentation_cli_contract.rs new file mode 100644 index 0000000..fe8d20e --- /dev/null +++ b/xtask/tests/documentation_cli_contract.rs @@ -0,0 +1,56 @@ +//! Subprocess contract for repository-wide verification with hermetic tools. + +#![cfg(all(feature = "repository-tasks", unix))] + +#[path = "cli_contract/documentation_tools.rs"] +mod documentation_tools; + +use std::io; + +const DOCUMENTATION_ERROR_DISPLAY: &str = + include_str!("../src/documentation_integrity/error/display.rs"); +const WORKFLOW_CONTRACT: &str = include_str!("../src/documentation_integrity/workflow_contract.rs"); + +#[test] +fn documentation_error_formatter_stays_below_the_hard_function_limit() -> Result<(), &'static str> { + let (_, after_signature) = DOCUMENTATION_ERROR_DISPLAY + .split_once(" fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {") + .ok_or("display implementation must retain its formatter")?; + let (body, _) = after_signature + .split_once("\n }\n}\n\nfn ") + .ok_or("display formatter must remain a directly inspectable function")?; + assert!( + body.lines().count() <= 59, + "DocumentationError::fmt exceeds the 60-line hard limit" + ); + Ok(()) +} + +#[test] +fn workflow_admission_parser_stays_below_the_hard_function_limit() -> Result<(), &'static str> { + let (_, after_signature) = WORKFLOW_CONTRACT + .split_once( + "fn admitted_steps(workflow: &str) -> Result, DocumentationError> {", + ) + .ok_or("workflow contract must retain its parser")?; + let (body, _) = after_signature + .split_once("\n}\n\nfn triggers_are_reviewed") + .ok_or("workflow parser must remain a directly inspectable function")?; + assert!( + body.lines().count() <= 59, + "admitted_steps exceeds the 60-line hard limit" + ); + Ok(()) +} + +#[test] +fn successful_verification_runs_every_documentation_tool_silently() -> Result<(), io::Error> { + let tools = documentation_tools::DocumentationTools::create()?; + let output = tools.invoke(&["verify"])?; + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + assert!(output.stderr.is_empty()); + tools.require_every_tool()?; + tools.close()?; + Ok(()) +} diff --git a/xtask/tests/repository_json_fuzz.rs b/xtask/tests/repository_json_fuzz.rs new file mode 100644 index 0000000..5894fd2 --- /dev/null +++ b/xtask/tests/repository_json_fuzz.rs @@ -0,0 +1,36 @@ +//! Integration laws for duplicate-refusing repository JSON fuzz admission. + +#![cfg(feature = "repository-json-fuzz")] + +use xtask::{RepositoryJsonAdmission, admit_repository_json}; + +#[test] +fn repository_json_fuzz_boundary_reaches_success_and_refusal_classes() { + let valid = br#"{"outer":[true,null,{"key":"value"}]}"#; + let malformed = br#"{"outer":["#; + let nested_duplicate = br#"{"outer":[{"key":1,"key":2}]}"#; + + assert_eq!( + admit_repository_json(valid), + RepositoryJsonAdmission::Admitted + ); + for refused in [malformed.as_slice(), nested_duplicate.as_slice()] { + assert_eq!( + admit_repository_json(refused), + RepositoryJsonAdmission::Refused + ); + } +} + +#[test] +fn repository_json_fuzz_boundary_refuses_resource_limits() { + let oversized = vec![b' '; 1_048_577]; + let deeply_nested = format!("{}null{}", "[".repeat(256), "]".repeat(256)); + + for refused in [oversized.as_slice(), deeply_nested.as_bytes()] { + assert_eq!( + admit_repository_json(refused), + RepositoryJsonAdmission::Refused + ); + } +} diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index c4faadc..f1d9efb 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -1,8 +1,24 @@ //! Written-policy regression evidence for the executable source-size law. const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); -const SOURCE_FILE: &str = include_str!("../src/source_structure/source_file.rs"); +const BOUNDED_PROCESS: &str = include_str!("../src/bounded_process.rs"); +const BOUNDED_PROCESS_CAPTURE: &str = include_str!("../src/bounded_process/capture.rs"); +const BOUNDED_PROCESS_CAPTURE_LIMIT: &str = include_str!("../src/bounded_process/capture_limit.rs"); +const BOUNDED_PROCESS_ERROR: &str = include_str!("../src/bounded_process/error.rs"); +const BOUNDED_PROCESS_INTERRUPT: &str = include_str!("../src/bounded_process/interrupt.rs"); +const BOUNDED_PROCESS_GROUP: &str = include_str!("../src/bounded_process/process_group.rs"); +const BOUNDED_PROCESS_GROUP_TESTS: &str = + include_str!("../src/bounded_process/process_group/tests.rs"); +const BOUNDED_PROCESS_READER: &str = include_str!("../src/bounded_process/reader.rs"); +const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); +const REPOSITORY_FIXTURE: &str = include_str!("../src/repository_fixture.rs"); +const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); +const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs"); +const GIT_PROCESS: &str = include_str!("../src/git_inventory/process.rs"); +const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); +const SOURCE_PURE_RUST_TESTS: &str = include_str!("../src/source_structure/pure_rust_tests.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); +const SOURCE_INVENTORY: &str = include_str!("../src/source_structure/source_inventory.rs"); #[test] fn written_source_limit_matches_the_executable_law() { @@ -16,7 +32,234 @@ fn written_source_limit_matches_the_executable_law() { } #[test] -fn source_root_identity_declares_its_unix_scope() { - assert!(SOURCE_FILE.contains("intentionally supported only on Unix hosts")); - assert!(SOURCE_FILE.contains("Unix device and inode identity")); +fn repository_file_admission_declares_its_unix_scope() { + assert!(REPOSITORY_FILE.contains("intentionally supported only on Unix hosts")); + assert!(REPOSITORY_FILE.contains("Unix device and inode identity")); +} + +#[test] +fn source_scan_revalidates_repository_identity_after_reading() { + assert_eq!( + SOURCE_STRUCTURE + .matches("verify_source_root(&source_root, repository_root)?;") + .count(), + 3 + ); +} + +#[test] +fn source_inventory_uses_the_admitted_repository_directory() { + assert!(SOURCE_STRUCTURE.contains(".process_directory()")); + assert!(SOURCE_INVENTORY.contains("paths_with(")); + assert!(!SOURCE_INVENTORY.contains("paths as git_paths")); + assert!(!GIT_PROCESS.contains("current_dir(")); +} + +#[test] +fn regular_source_admission_has_one_refusal_boundary() { + assert_eq!( + SOURCE_STRUCTURE + .matches("SourceFileAdmission::NonRegular") + .count(), + 1 + ); + assert_eq!(SOURCE_STRUCTURE.matches("fn admit_regular(").count(), 1); +} + +#[test] +fn sanitized_git_fixture_has_one_process_authority() { + let definitions = REPOSITORY_FIXTURE.matches("fn run_git(").count() + + SOURCE_PURE_RUST_TESTS.matches("fn run_git(").count(); + assert_eq!(definitions, 1); +} + +#[test] +fn git_inventory_uses_the_deadline_bounded_process_layer() { + assert!(GIT_PROCESS.contains("const GIT_DEADLINE: Duration")); + assert!(GIT_PROCESS.contains("bounded_process::capture_with_limits(")); +} + +#[test] +fn captured_process_keeps_the_group_leader_until_reader_collection_finishes() +-> Result<(), &'static str> { + let (_, after_signature) = BOUNDED_PROCESS_CAPTURE + .split_once(" fn finish(") + .ok_or("captured process must retain its finish boundary")?; + let (body, _) = after_signature + .split_once("\n fn cleanup_readers") + .ok_or("captured process finish boundary must remain inspectable")?; + let wait = body + .find("wait_for_child") + .ok_or("captured process must reap its child")?; + + for operation in [ + "self.stdout.receive", + "self.stderr.receive", + "self.stdout.join", + "self.stderr.join", + ] { + let position = body + .find(operation) + .ok_or("captured process must collect and join both output streams")?; + assert!(position < wait, "child wait precedes {operation}"); + } + assert!( + !body + .get(wait..) + .unwrap_or_default() + .contains("cleanup_process"), + "cleanup may group-kill after the child ownership lifetime ends" + ); + Ok(()) +} + +#[test] +fn descendant_cleanup_uses_disconnect_evidence_instead_of_elapsed_time() { + assert!(!BOUNDED_PROCESS_GROUP_TESTS.contains("descendant_survived_cleanup")); + assert!(!BOUNDED_PROCESS_GROUP_TESTS.contains("Duration::from_millis(500)")); + assert!(BOUNDED_PROCESS_GROUP_TESTS.contains("require_descendant_disconnect")); +} + +#[test] +fn repository_git_fixtures_use_the_bounded_process_layer() { + assert!(REPOSITORY_FIXTURE.contains("bounded_process::status(")); + assert!(REPOSITORY_FIXTURE.contains("GIT_FIXTURE_DEADLINE")); + assert!(!REPOSITORY_FIXTURE.contains(".output()")); +} + +#[test] +fn repository_git_fixtures_clear_the_ambient_environment() { + assert!(REPOSITORY_FIXTURE.contains(".env_clear()")); + assert!(REPOSITORY_FIXTURE.contains("env::var_os(\"PATH\")")); + assert!(REPOSITORY_FIXTURE.contains(".env(\"PATH\"")); + assert!(REPOSITORY_FIXTURE.contains(".env(\"LC_ALL\", \"C\")")); +} + +#[test] +fn process_fixtures_do_not_write_to_rust_stdout() { + assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); +} + +#[test] +fn process_fixtures_isolate_git_from_host_configuration() { + assert_eq!( + BOUNDED_PROCESS_TESTS + .matches("Command::new(\"git\")") + .count(), + 1 + ); + assert!(BOUNDED_PROCESS_TESTS.contains(".env_clear()")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"PATH\"")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_NOSYSTEM\", \"1\")")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_GLOBAL\"")); + assert!(BOUNDED_PROCESS_TESTS.contains("--template=")); + assert!(!BOUNDED_PROCESS_TESTS.contains("template.display()")); +} + +#[test] +fn repository_process_boundaries_document_every_exported_contract() -> Result<(), String> { + require_docs( + BOUNDED_PROCESS, + &["pub(crate) struct ProcessOutput", "pub(crate) fn status("], + )?; + require_docs( + BOUNDED_PROCESS_CAPTURE, + &[ + "pub(crate) fn capture(", + "pub(crate) fn capture_with(", + "pub(crate) fn capture_with_limits(", + ], + )?; + require_docs( + BOUNDED_PROCESS_CAPTURE_LIMIT, + &[ + "pub(crate) struct CaptureLimits", + " pub(crate) const fn new(", + ], + )?; + require_docs( + BOUNDED_PROCESS_ERROR, + &[ + "pub(crate) enum ProcessError", + " Additional {", + " Cleanup {", + " Io {", + " Interrupted {", + " MissingStream {", + " OutputLimit {", + " ReaderPanic {", + " Timeout {", + " pub(crate) fn is_not_found(", + ], + )?; + require_docs( + BOUNDED_PROCESS_INTERRUPT, + &[ + "pub(super) struct InterruptGuard", + " pub(super) fn begin(", + " pub(super) fn refusal(", + ], + )?; + require_docs( + BOUNDED_PROCESS_GROUP, + &[ + "pub(super) struct ProcessGroup", + " pub(super) fn for_child(", + " pub(super) fn terminate(", + ], + )?; + require_docs( + BOUNDED_PROCESS_READER, + &[ + "pub(super) struct ReaderWorker", + " pub(super) fn start(", + " pub(super) fn receive(", + " pub(super) fn join(", + ], + )?; + require_docs(REPOSITORY_FIXTURE, &["pub(crate) fn run_git("])?; + require_docs( + GIT_INVENTORY_ERROR, + &[ + "pub(crate) enum GitOutputUnit", + " Bytes,", + " Items,", + "pub(crate) enum GitInventoryError", + " DuplicatePath(", + " EmptyPath {", + " Failed {", + " DiagnosticEncoding {", + " OutputBound {", + " OutputFraming {", + " Process {", + " Run {", + ], + )?; + require_docs( + GIT_PATH_STREAM, + &[ + "pub(super) fn read_paths(", + "pub(crate) struct GitPath(", + " pub(crate) const fn new(", + " pub(crate) fn as_bytes(", + ], + )?; + require_docs(GIT_PROCESS, &["pub(crate) fn paths_with("]) +} + +fn require_docs(source: &str, declarations: &[&str]) -> Result<(), String> { + for declaration in declarations { + let (before, _) = source + .split_once(declaration) + .ok_or_else(|| format!("missing declaration `{declaration}`"))?; + let documented = before + .lines() + .rev() + .find(|line| !line.trim_start().starts_with("#[")) + .is_some_and(|line| line.trim_start().starts_with("///")); + if !documented { + return Err(format!("missing rustdoc for `{declaration}`")); + } + } + Ok(()) }