From 9216a95a9604a468b57234c1f3b704acc779aa7b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 11:46:05 -0700 Subject: [PATCH 001/113] Extract bounded Git path inventory --- xtask/src/git_inventory.rs | 9 + xtask/src/git_inventory/error.rs | 155 ++++++++++++++++++ .../path_stream.rs} | 77 +++++---- .../path_stream}/tests.rs | 14 +- .../process.rs} | 46 +++--- .../process}/tests.rs | 4 +- xtask/src/main.rs | 5 + xtask/src/source_structure.rs | 13 +- xtask/src/source_structure/source_error.rs | 135 ++------------- xtask/src/source_structure/tests.rs | 34 ++-- 10 files changed, 273 insertions(+), 219 deletions(-) create mode 100644 xtask/src/git_inventory.rs create mode 100644 xtask/src/git_inventory/error.rs rename xtask/src/{source_structure/git_path_stream.rs => git_inventory/path_stream.rs} (62%) rename xtask/src/{source_structure/git_path_stream => git_inventory/path_stream}/tests.rs (88%) rename xtask/src/{source_structure/git_path_inventory.rs => git_inventory/process.rs} (75%) rename xtask/src/{source_structure/git_path_inventory => git_inventory/process}/tests.rs (86%) diff --git a/xtask/src/git_inventory.rs b/xtask/src/git_inventory.rs new file mode 100644 index 0000000..5ca55ca --- /dev/null +++ b/xtask/src/git_inventory.rs @@ -0,0 +1,9 @@ +//! This module owns bounded, deterministic Git path inventory. + +mod error; +mod path_stream; +mod process; + +pub(crate) use error::{GitInventoryError, GitOutputUnit}; +pub(crate) use path_stream::GitPath; +pub(crate) use process::paths; diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs new file mode 100644 index 0000000..21017e5 --- /dev/null +++ b/xtask/src/git_inventory/error.rs @@ -0,0 +1,155 @@ +//! This module owns typed Git inventory failures and stable diagnostics. + +use std::error::Error; +use std::fmt::{self, Write as _}; +use std::io; +use std::string::FromUtf8Error; + +use crate::diagnostic::escaped_controls; + +#[derive(Clone, Copy)] +pub(crate) enum GitOutputUnit { + Bytes, + Items, +} + +pub(crate) enum GitInventoryError { + DuplicatePath(Vec), + Failed { + operation: &'static str, + code: Option, + stderr: String, + }, + DiagnosticEncoding { + operation: &'static str, + code: Option, + source: FromUtf8Error, + }, + OutputBound { + operation: &'static str, + stream: &'static str, + maximum: usize, + unit: GitOutputUnit, + }, + OutputFraming { + operation: &'static str, + }, + Pipe { + operation: &'static str, + stream: &'static str, + }, + Run { + operation: &'static str, + action: &'static str, + source: io::Error, + }, + Worker { + operation: &'static str, + }, +} + +impl fmt::Debug for GitInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for GitInventoryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DuplicatePath(path) => { + formatter.write_str("git returned duplicate path `")?; + escaped_bytes(formatter, path)?; + formatter.write_str("`") + } + Self::Failed { + operation, + code, + stderr, + } => git_failed(formatter, operation, *code, stderr), + Self::DiagnosticEncoding { + operation, code, .. + } => write!( + formatter, + "`{operation}` failed with code {code:?} and returned non-UTF-8 diagnostics" + ), + Self::OutputBound { + operation, + stream, + maximum, + unit, + } => write!( + formatter, + "`{operation}` exceeded the {stream} bound of {maximum} {}", + unit.label() + ), + Self::OutputFraming { operation } => { + write!( + formatter, + "`{operation}` returned a non-NUL-terminated path" + ) + } + Self::Pipe { operation, stream } => { + write!(formatter, "`{operation}` did not provide its {stream} pipe") + } + Self::Run { + operation, action, .. + } => write!(formatter, "cannot {action} `{operation}`"), + Self::Worker { operation } => { + write!( + formatter, + "`{operation}` diagnostic reader stopped unexpectedly" + ) + } + } + } +} + +impl GitOutputUnit { + const fn label(self) -> &'static str { + match self { + Self::Bytes => "bytes", + Self::Items => "items", + } + } +} + +impl Error for GitInventoryError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::DiagnosticEncoding { source, .. } => Some(source), + Self::Run { source, .. } => Some(source), + Self::DuplicatePath(_) + | Self::Failed { .. } + | Self::OutputBound { .. } + | Self::OutputFraming { .. } + | Self::Pipe { .. } + | Self::Worker { .. } => None, + } + } +} + +fn git_failed( + formatter: &mut fmt::Formatter<'_>, + operation: &str, + code: Option, + stderr: &str, +) -> fmt::Result { + write!(formatter, "`{operation}` failed with code {code:?}")?; + if stderr.is_empty() { + return Ok(()); + } + formatter.write_str(": ")?; + escaped_controls(formatter, stderr.trim_end()) +} + +fn escaped_bytes(formatter: &mut fmt::Formatter<'_>, bytes: &[u8]) -> fmt::Result { + for byte in bytes { + if byte.is_ascii_graphic() || *byte == b' ' { + formatter.write_char(char::from(*byte))?; + } else { + write!(formatter, "\\x{byte:02x}")?; + } + } + Ok(()) +} diff --git a/xtask/src/source_structure/git_path_stream.rs b/xtask/src/git_inventory/path_stream.rs similarity index 62% rename from xtask/src/source_structure/git_path_stream.rs rename to xtask/src/git_inventory/path_stream.rs index 50d1ba0..a95c81a 100644 --- a/xtask/src/source_structure/git_path_stream.rs +++ b/xtask/src/git_inventory/path_stream.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use std::io::Read; -use super::{GitOutputUnit, SourceStructureError}; +use super::{GitInventoryError, GitOutputUnit}; const GIT_PATH_LIMITS: GitPathLimits = GitPathLimits { path_bytes: 4_096, @@ -21,7 +21,7 @@ struct GitPathLimits { pub(super) fn read_paths( reader: impl Read, operation: &'static str, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { read_paths_with(reader, operation, GIT_PATH_LIMITS) } @@ -29,13 +29,13 @@ fn read_paths_with( mut reader: impl Read, operation: &'static str, limits: GitPathLimits, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { let mut decoder = GitPathDecoder::new(operation, limits); let mut buffer = [0_u8; 4_096]; loop { let read = reader .read(&mut buffer) - .map_err(|source| SourceStructureError::RunGit { + .map_err(|source| GitInventoryError::Run { operation, action: "read paths from", source, @@ -43,14 +43,12 @@ fn read_paths_with( if read == 0 { break; } - let bytes = buffer - .get(..read) - .ok_or(SourceStructureError::GitOutputBound { - operation, - stream: "path read", - maximum: buffer.len(), - unit: GitOutputUnit::Bytes, - })?; + let bytes = buffer.get(..read).ok_or(GitInventoryError::OutputBound { + operation, + stream: "path read", + maximum: buffer.len(), + unit: GitOutputUnit::Bytes, + })?; decoder.admit(bytes)?; } decoder.finish() @@ -62,18 +60,18 @@ struct GitPathDecoder { observed_bytes: usize, observed_paths: usize, operation: &'static str, - paths: BTreeSet, + paths: BTreeSet, } #[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] -pub(super) struct GitPathRecord(Vec); +pub(crate) struct GitPath(Vec); -impl GitPathRecord { - pub(super) const fn new(bytes: Vec) -> Self { +impl GitPath { + pub(crate) const fn new(bytes: Vec) -> Self { Self(bytes) } - pub(super) fn as_bytes(&self) -> &[u8] { + pub(crate) fn as_bytes(&self) -> &[u8] { &self.0 } } @@ -90,17 +88,18 @@ impl GitPathDecoder { } } - fn admit(&mut self, bytes: &[u8]) -> Result<(), SourceStructureError> { - self.observed_bytes = self.observed_bytes.checked_add(bytes.len()).ok_or( - SourceStructureError::GitOutputBound { - operation: self.operation, - stream: "path stream bytes", - maximum: self.limits.stream_bytes, - unit: GitOutputUnit::Bytes, - }, - )?; + fn admit(&mut self, bytes: &[u8]) -> Result<(), GitInventoryError> { + self.observed_bytes = + self.observed_bytes + .checked_add(bytes.len()) + .ok_or(GitInventoryError::OutputBound { + operation: self.operation, + stream: "path stream bytes", + maximum: self.limits.stream_bytes, + unit: GitOutputUnit::Bytes, + })?; if self.observed_bytes > self.limits.stream_bytes { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path stream bytes", maximum: self.limits.stream_bytes, @@ -113,12 +112,12 @@ impl GitPathDecoder { Ok(()) } - fn admit_byte(&mut self, byte: u8) -> Result<(), SourceStructureError> { + fn admit_byte(&mut self, byte: u8) -> Result<(), GitInventoryError> { if byte == 0 { return self.admit_path(); } if self.current.len() >= self.limits.path_bytes { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path bytes", maximum: self.limits.path_bytes, @@ -129,44 +128,42 @@ impl GitPathDecoder { Ok(()) } - fn admit_path(&mut self) -> Result<(), SourceStructureError> { + fn admit_path(&mut self) -> Result<(), GitInventoryError> { if self.current.is_empty() { - return Err(SourceStructureError::GitOutputFraming { + return Err(GitInventoryError::OutputFraming { operation: self.operation, }); } self.observed_paths = self.observed_paths .checked_add(1) - .ok_or(SourceStructureError::GitOutputBound { + .ok_or(GitInventoryError::OutputBound { operation: self.operation, stream: "path count", maximum: self.limits.paths, unit: GitOutputUnit::Items, })?; if self.observed_paths > self.limits.paths { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation: self.operation, stream: "path count", maximum: self.limits.paths, unit: GitOutputUnit::Items, }); } - let path = GitPathRecord::new(std::mem::take(&mut self.current)); + let path = GitPath::new(std::mem::take(&mut self.current)); if self.paths.insert(path.clone()) { Ok(()) } else { - Err(SourceStructureError::DuplicatePath( - path.as_bytes().to_vec(), - )) + Err(GitInventoryError::DuplicatePath(path.as_bytes().to_vec())) } } - fn finish(self) -> Result, SourceStructureError> { + fn finish(self) -> Result, GitInventoryError> { if self.current.is_empty() { Ok(self.paths) } else { - Err(SourceStructureError::GitOutputFraming { + Err(GitInventoryError::OutputFraming { operation: self.operation, }) } @@ -174,5 +171,5 @@ impl GitPathDecoder { } #[cfg(test)] -#[path = "git_path_stream/tests.rs"] +#[path = "path_stream/tests.rs"] mod tests; diff --git a/xtask/src/source_structure/git_path_stream/tests.rs b/xtask/src/git_inventory/path_stream/tests.rs similarity index 88% rename from xtask/src/source_structure/git_path_stream/tests.rs rename to xtask/src/git_inventory/path_stream/tests.rs index 29ead53..96eec42 100644 --- a/xtask/src/source_structure/git_path_stream/tests.rs +++ b/xtask/src/git_inventory/path_stream/tests.rs @@ -2,7 +2,7 @@ use std::io::Cursor; -use super::{GitPathLimits, SourceStructureError, read_paths_with}; +use super::{GitInventoryError, GitPathLimits, read_paths_with}; const TEST_LIMITS: GitPathLimits = GitPathLimits { path_bytes: 3, @@ -15,7 +15,7 @@ fn git_path_stream_refuses_an_oversized_path_without_buffering_the_tail() { let result = read_paths_with(Cursor::new(b"abcd\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path bytes", maximum: 3, .. @@ -28,7 +28,7 @@ fn git_path_stream_refuses_an_oversized_inventory() { let result = read_paths_with(Cursor::new(b"a\0b\0c\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path count", maximum: 2, .. @@ -41,7 +41,7 @@ fn git_path_stream_refuses_excess_framing_bytes() { let result = read_paths_with(Cursor::new(b"\0\0\0\0\0\0\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputBound { + Err(GitInventoryError::OutputBound { stream: "path stream bytes", maximum: 6, .. @@ -54,7 +54,7 @@ fn git_path_stream_requires_nul_framing() { let result = read_paths_with(Cursor::new(b"abc"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputFraming { + Err(GitInventoryError::OutputFraming { operation: "test paths" }) )); @@ -66,7 +66,7 @@ fn git_path_stream_refuses_empty_records() { let result = read_paths_with(Cursor::new(stream), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::GitOutputFraming { + Err(GitInventoryError::OutputFraming { operation: "test paths" }) )); @@ -78,7 +78,7 @@ fn git_path_stream_refuses_duplicate_records() { let result = read_paths_with(Cursor::new(b"a\0a\0"), "test paths", TEST_LIMITS); assert!(matches!( result, - Err(SourceStructureError::DuplicatePath(ref path)) if path == b"a" + Err(GitInventoryError::DuplicatePath(ref path)) if path == b"a" )); } diff --git a/xtask/src/source_structure/git_path_inventory.rs b/xtask/src/git_inventory/process.rs similarity index 75% rename from xtask/src/source_structure/git_path_inventory.rs rename to xtask/src/git_inventory/process.rs index 5855ab4..65475c3 100644 --- a/xtask/src/source_structure/git_path_inventory.rs +++ b/xtask/src/git_inventory/process.rs @@ -1,4 +1,4 @@ -//! This module owns bounded Git path streaming and diagnostic collection. +//! This module owns bounded Git path process execution and collection. use std::collections::BTreeSet; use std::io; @@ -8,8 +8,8 @@ use std::thread::{self, JoinHandle}; use crate::process_output::{BoundedBytes, bounded_bytes}; -use super::git_path_stream::{GitPathRecord, read_paths}; -use super::{GitOutputUnit, SourceStructureError}; +use super::path_stream::{GitPath, read_paths}; +use super::{GitInventoryError, GitOutputUnit}; const GIT_DIAGNOSTIC_LIMIT_BYTES: usize = 65_536; @@ -25,11 +25,11 @@ struct GitProcess { /// Collection reads standard output before requesting termination, waits for /// the child before joining the diagnostic reader, and then preserves the /// established error precedence. -pub(super) fn git_paths( +pub(crate) fn paths( repository_root: &Path, arguments: &[&str], operation: &'static str, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { let process = start_git(repository_root, arguments, operation)?; let paths = read_paths(process.stdout, operation); collect_git_result(process.child, process.diagnostic_worker, paths, operation) @@ -39,28 +39,28 @@ fn start_git( repository_root: &Path, arguments: &[&str], operation: &'static str, -) -> Result { +) -> Result { let mut child = Command::new("git") .args(arguments) .current_dir(repository_root) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .map_err(|source| SourceStructureError::RunGit { + .map_err(|source| GitInventoryError::Run { operation, action: "start", source, })?; let Some(stdout) = child.stdout.take() else { cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::GitPipe { + return Err(GitInventoryError::Pipe { operation, stream: "stdout", }); }; let Some(stderr) = child.stderr.take() else { cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::GitPipe { + return Err(GitInventoryError::Pipe { operation, stream: "stderr", }); @@ -72,7 +72,7 @@ fn start_git( Ok(worker) => worker, Err(source) => { cleanup_child(&mut child, operation)?; - return Err(SourceStructureError::RunGit { + return Err(GitInventoryError::Run { operation, action: "start the diagnostic reader for", source, @@ -89,33 +89,33 @@ fn start_git( fn collect_git_result( mut child: Child, diagnostic_worker: JoinHandle>, - paths: Result, SourceStructureError>, + paths: Result, GitInventoryError>, operation: &'static str, -) -> Result, SourceStructureError> { +) -> Result, GitInventoryError> { let stop = if paths.is_err() { request_stop(&mut child, operation) } else { Ok(()) }; - let status = child.wait().map_err(|source| SourceStructureError::RunGit { + let status = child.wait().map_err(|source| GitInventoryError::Run { operation, action: "wait for", source, }); let diagnostic = diagnostic_worker .join() - .map_err(|_| SourceStructureError::GitWorker { operation })?; + .map_err(|_| GitInventoryError::Worker { operation })?; stop?; let paths = paths?; let status = status?; - let diagnostic = diagnostic.map_err(|source| SourceStructureError::RunGit { + let diagnostic = diagnostic.map_err(|source| GitInventoryError::Run { operation, action: "read diagnostics from", source, })?; if diagnostic.exceeded { - return Err(SourceStructureError::GitOutputBound { + return Err(GitInventoryError::OutputBound { operation, stream: "diagnostic bytes", maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, @@ -128,7 +128,7 @@ fn collect_git_result( Ok(paths) } -fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), SourceStructureError> { +fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), GitInventoryError> { child .kill() .or_else(|source| { @@ -138,16 +138,16 @@ fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), Source Err(source) } }) - .map_err(|source| SourceStructureError::RunGit { + .map_err(|source| GitInventoryError::Run { operation, action: "stop", source, }) } -fn cleanup_child(child: &mut Child, operation: &'static str) -> Result<(), SourceStructureError> { +fn cleanup_child(child: &mut Child, operation: &'static str) -> Result<(), GitInventoryError> { let stop = request_stop(child, operation); - let wait = child.wait().map_err(|source| SourceStructureError::RunGit { + let wait = child.wait().map_err(|source| GitInventoryError::Run { operation, action: "wait for", source, @@ -160,14 +160,14 @@ fn git_failure( operation: &'static str, code: Option, diagnostic: Vec, -) -> SourceStructureError { +) -> GitInventoryError { match String::from_utf8(diagnostic) { - Ok(stderr) => SourceStructureError::GitFailed { + Ok(stderr) => GitInventoryError::Failed { operation, code, stderr, }, - Err(source) => SourceStructureError::GitDiagnosticEncoding { + Err(source) => GitInventoryError::DiagnosticEncoding { operation, code, source, diff --git a/xtask/src/source_structure/git_path_inventory/tests.rs b/xtask/src/git_inventory/process/tests.rs similarity index 86% rename from xtask/src/source_structure/git_path_inventory/tests.rs rename to xtask/src/git_inventory/process/tests.rs index 6233753..1cfb253 100644 --- a/xtask/src/source_structure/git_path_inventory/tests.rs +++ b/xtask/src/git_inventory/process/tests.rs @@ -2,7 +2,7 @@ use std::io::Cursor; -use super::{SourceStructureError, git_failure}; +use super::{GitInventoryError, git_failure}; use crate::process_output::bounded_bytes; #[test] @@ -10,7 +10,7 @@ fn git_diagnostic_encoding_failure_retains_exit_status() { let error = git_failure("test diagnostics", Some(9), vec![u8::MAX]); assert!(matches!( error, - SourceStructureError::GitDiagnosticEncoding { + GitInventoryError::DiagnosticEncoding { operation: "test diagnostics", code: Some(9), .. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2635b98..3afc2d8 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -19,6 +19,11 @@ mod fuzz_campaign; reason = "the parent command dispatcher is the only consumer" )] mod fuzz_seed_corpus; +#[allow( + clippy::redundant_pub_crate, + reason = "bounded Git inventory is shared by sibling repository tasks" +)] +mod git_inventory; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index df78614..8aa3ad5 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -1,7 +1,5 @@ //! This module owns source inventory orchestration and the 500-line law. -mod git_path_inventory; -mod git_path_stream; mod repository_path; mod source_error; mod source_file; @@ -10,10 +8,9 @@ use std::collections::BTreeSet; use std::io::{self, BufRead, BufReader}; use std::path::Path; -use git_path_inventory::git_paths; -use git_path_stream::GitPathRecord; +use crate::git_inventory::{GitPath, paths as git_paths}; use repository_path::RepositoryPath; -pub(super) use source_error::{GitOutputUnit, SourceStructureError}; +pub(super) use source_error::SourceStructureError; use source_file::{OpenSourceError, SourceRoot}; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; @@ -76,8 +73,8 @@ fn source_paths(repository_root: &Path) -> Result, SourceStr } fn select_source_paths( - present: &BTreeSet, - deleted: &BTreeSet, + present: &BTreeSet, + deleted: &BTreeSet, ) -> Result, SourceStructureError> { present .difference(deleted) @@ -86,7 +83,7 @@ fn select_source_paths( .collect() } -fn admit_source_path(path: &GitPathRecord) -> Result { +fn admit_source_path(path: &GitPath) -> Result { let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { SourceStructureError::GitPathEncoding { operation: "source path admission", diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 1200e59..4fb432b 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -1,51 +1,20 @@ //! This module owns typed source-structure failures and stable diagnostics. use std::error::Error; -use std::fmt::{self, Write as _}; +use std::fmt; use std::io; use std::path::PathBuf; use std::string::FromUtf8Error; use crate::diagnostic::{escaped_controls, escaped_path}; - -#[derive(Clone, Copy)] -pub(crate) enum GitOutputUnit { - Bytes, - Items, -} +use crate::git_inventory::GitInventoryError; pub(crate) enum SourceStructureError { - DuplicatePath(Vec), - GitFailed { - operation: &'static str, - code: Option, - stderr: String, - }, - GitDiagnosticEncoding { - operation: &'static str, - code: Option, - source: FromUtf8Error, - }, + GitInventory(GitInventoryError), GitPathEncoding { operation: &'static str, source: FromUtf8Error, }, - GitOutputBound { - operation: &'static str, - stream: &'static str, - maximum: usize, - unit: GitOutputUnit, - }, - GitOutputFraming { - operation: &'static str, - }, - GitPipe { - operation: &'static str, - stream: &'static str, - }, - GitWorker { - operation: &'static str, - }, Inspect { path: PathBuf, source: io::Error, @@ -53,11 +22,6 @@ pub(crate) enum SourceStructureError { InvalidPath(String), NonRegular(PathBuf), RepositoryRootChanged(PathBuf), - RunGit { - operation: &'static str, - action: &'static str, - source: io::Error, - }, Violations { maximum: u64, paths: Vec, @@ -73,50 +37,10 @@ impl fmt::Debug for SourceStructureError { impl fmt::Display for SourceStructureError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::DuplicatePath(path) => { - formatter.write_str("git returned duplicate path `")?; - escaped_bytes(formatter, path)?; - formatter.write_str("`") - } - Self::GitFailed { - operation, - code, - stderr, - } => git_failed(formatter, operation, *code, stderr), - Self::GitDiagnosticEncoding { - operation, code, .. - } => write!( - formatter, - "`{operation}` failed with code {code:?} and returned non-UTF-8 diagnostics" - ), + Self::GitInventory(error) => write!(formatter, "{error}"), Self::GitPathEncoding { operation, .. } => { write!(formatter, "`{operation}` returned a non-UTF-8 path") } - Self::GitOutputBound { - operation, - stream, - maximum, - unit, - } => write!( - formatter, - "`{operation}` exceeded the {stream} bound of {maximum} {}", - unit.label() - ), - Self::GitOutputFraming { operation } => { - write!( - formatter, - "`{operation}` returned a non-NUL-terminated path" - ) - } - Self::GitPipe { operation, stream } => { - write!(formatter, "`{operation}` did not provide its {stream} pipe") - } - Self::GitWorker { operation } => { - write!( - formatter, - "`{operation}` diagnostic reader stopped unexpectedly" - ) - } Self::Inspect { path, .. } => { formatter.write_str("cannot inspect `")?; escaped_path(formatter, path)?; @@ -137,37 +61,18 @@ impl fmt::Display for SourceStructureError { escaped_path(formatter, path)?; formatter.write_str("`") } - Self::RunGit { - operation, action, .. - } => write!(formatter, "cannot {action} `{operation}`"), Self::Violations { maximum, paths } => violations_display(formatter, *maximum, paths), } } } -impl GitOutputUnit { - const fn label(self) -> &'static str { - match self { - Self::Bytes => "bytes", - Self::Items => "items", - } - } -} - impl Error for SourceStructureError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::GitDiagnosticEncoding { source, .. } | Self::GitPathEncoding { source, .. } => { - Some(source) - } - Self::Inspect { source, .. } | Self::RunGit { source, .. } => Some(source), - Self::DuplicatePath(_) - | Self::GitFailed { .. } - | Self::GitOutputBound { .. } - | Self::GitOutputFraming { .. } - | Self::GitPipe { .. } - | Self::GitWorker { .. } - | Self::InvalidPath(_) + Self::GitInventory(error) => Some(error), + Self::GitPathEncoding { source, .. } => Some(source), + Self::Inspect { source, .. } => Some(source), + Self::InvalidPath(_) | Self::NonRegular(_) | Self::RepositoryRootChanged(_) | Self::Violations { .. } => None, @@ -175,28 +80,10 @@ impl Error for SourceStructureError { } } -fn git_failed( - formatter: &mut fmt::Formatter<'_>, - operation: &str, - code: Option, - stderr: &str, -) -> fmt::Result { - write!(formatter, "`{operation}` failed with code {code:?}: ")?; - escaped_controls(formatter, stderr.trim()) -} - -fn escaped_bytes(formatter: &mut fmt::Formatter<'_>, bytes: &[u8]) -> fmt::Result { - if let Ok(text) = std::str::from_utf8(bytes) { - return escaped_controls(formatter, text); - } - for byte in bytes { - if byte.is_ascii_graphic() || *byte == b' ' { - formatter.write_char(char::from(*byte))?; - } else { - write!(formatter, "\\x{byte:02x}")?; - } +impl From for SourceStructureError { + fn from(error: GitInventoryError) -> Self { + Self::GitInventory(error) } - Ok(()) } fn violations_display( diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index 1987083..eef2886 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -63,9 +63,11 @@ fn early_source_refusal_does_not_claim_an_exact_line_count() { #[test] fn source_structure_diagnostics_are_stable() { - let framing = super::SourceStructureError::GitOutputFraming { + use crate::git_inventory::{GitInventoryError, GitOutputUnit}; + + let framing = super::SourceStructureError::GitInventory(GitInventoryError::OutputFraming { operation: "git inventory", - }; + }); let non_regular = super::SourceStructureError::NonRegular("src/link.rs".into()); assert_eq!( framing.to_string(), @@ -83,18 +85,18 @@ fn source_structure_diagnostics_are_stable() { violations.to_string(), "repository source modules exceed the 7-line hard maximum; src/large.rs: >7" ); - let byte_bound = super::SourceStructureError::GitOutputBound { + let byte_bound = super::SourceStructureError::GitInventory(GitInventoryError::OutputBound { operation: "git inventory", stream: "path bytes", maximum: 4_096, - unit: super::GitOutputUnit::Bytes, - }; - let item_bound = super::SourceStructureError::GitOutputBound { + unit: GitOutputUnit::Bytes, + }); + let item_bound = super::SourceStructureError::GitInventory(GitInventoryError::OutputBound { operation: "git inventory", stream: "path count", maximum: 100_000, - unit: super::GitOutputUnit::Items, - }; + unit: GitOutputUnit::Items, + }); assert_eq!( byte_bound.to_string(), "`git inventory` exceeded the path bytes bound of 4096 bytes" @@ -107,11 +109,13 @@ fn source_structure_diagnostics_are_stable() { #[test] fn git_diagnostics_cannot_inject_terminal_control_lines() { - let error = super::SourceStructureError::GitFailed { + use crate::git_inventory::GitInventoryError; + + let error = super::SourceStructureError::GitInventory(GitInventoryError::Failed { operation: "git inventory", code: Some(9), stderr: String::from("first\nError: forged\rrewrite\u{1b}[31m"), - }; + }); let diagnostic = error.to_string(); assert_eq!( diagnostic, @@ -285,11 +289,11 @@ fn source_paths_refuse_host_dependent_spellings() { fn source_selection_refuses_an_unsafe_source_record() { use std::collections::BTreeSet; - use super::git_path_stream::GitPathRecord; + use crate::git_inventory::GitPath; let present = BTreeSet::from([ - GitPathRecord::new(b"../escape.rs".to_vec()), - GitPathRecord::new(b"notes/x:y".to_vec()), + GitPath::new(b"../escape.rs".to_vec()), + GitPath::new(b"notes/x:y".to_vec()), ]); let result = super::select_source_paths(&present, &BTreeSet::new()); assert!(matches!( @@ -302,9 +306,9 @@ fn source_selection_refuses_an_unsafe_source_record() { fn source_selection_refuses_a_non_utf8_source_record() { use std::collections::BTreeSet; - use super::git_path_stream::GitPathRecord; + use crate::git_inventory::GitPath; - let present = BTreeSet::from([GitPathRecord::new(b"bad\xff.rs".to_vec())]); + let present = BTreeSet::from([GitPath::new(b"bad\xff.rs".to_vec())]); let result = super::select_source_paths(&present, &BTreeSet::new()); assert!(matches!( result, From e61c508b9fd59eb67aff4dd36e379b65f67022b3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 11:50:43 -0700 Subject: [PATCH 002/113] Add Rust documentation corpus laws --- xtask/src/documentation_integrity.rs | 4 + xtask/src/documentation_integrity/corpus.rs | 174 +++++++++++++++++ .../documentation_integrity/corpus/tests.rs | 178 ++++++++++++++++++ xtask/src/documentation_integrity/error.rs | 81 ++++++++ xtask/src/main.rs | 2 + 5 files changed, 439 insertions(+) create mode 100644 xtask/src/documentation_integrity.rs create mode 100644 xtask/src/documentation_integrity/corpus.rs create mode 100644 xtask/src/documentation_integrity/corpus/tests.rs create mode 100644 xtask/src/documentation_integrity/error.rs diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs new file mode 100644 index 0000000..802d9c3 --- /dev/null +++ b/xtask/src/documentation_integrity.rs @@ -0,0 +1,4 @@ +//! This module owns documentation and workflow integrity orchestration. + +mod corpus; +mod error; diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs new file mode 100644 index 0000000..6a89664 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus.rs @@ -0,0 +1,174 @@ +//! This module owns deterministic documentation source selection. + +use std::fs; +use std::io; +use std::path::Path; + +use xtask::protocol_admission::posix_relative_path; + +use super::error::DocumentationError; +use crate::git_inventory::{GitPath, paths}; + +const MARKDOWN_PRESENT: [&str; 7] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + "*.md", +]; +const MARKDOWN_DELETED: [&str; 5] = ["ls-files", "-z", "--deleted", "--", "*.md"]; +const WORKFLOW_PRESENT: [&str; 8] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; +const WORKFLOW_DELETED: [&str; 6] = [ + "ls-files", + "-z", + "--deleted", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; + +pub(super) struct SourceCorpus { + paths: Vec, +} + +#[derive(Clone, Copy)] +enum CorpusKind { + Markdown, + Workflow, +} + +impl SourceCorpus { + pub(super) fn markdown(repository_root: &Path) -> Result { + Self::read(repository_root, CorpusKind::Markdown) + } + + pub(super) fn workflow(repository_root: &Path) -> Result { + Self::read(repository_root, CorpusKind::Workflow) + } + + pub(super) fn paths(&self) -> &[String] { + &self.paths + } + + fn read(repository_root: &Path, kind: CorpusKind) -> Result { + let present = paths( + repository_root, + kind.present_arguments(), + kind.present_operation(), + )?; + let deleted = paths( + repository_root, + kind.deleted_arguments(), + kind.deleted_operation(), + )?; + let selected = present.difference(&deleted); + let paths = admit_paths(repository_root, selected, kind)?; + if paths.is_empty() { + Err(DocumentationError::EmptyCorpus(kind.label())) + } else { + Ok(Self { paths }) + } + } +} + +impl CorpusKind { + const fn label(self) -> &'static str { + match self { + Self::Markdown => "Markdown", + Self::Workflow => "GitHub Actions workflow", + } + } + + const fn present_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_PRESENT, + Self::Workflow => &WORKFLOW_PRESENT, + } + } + + const fn deleted_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_DELETED, + Self::Workflow => &WORKFLOW_DELETED, + } + } + + const fn present_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown present paths", + Self::Workflow => "git workflow present paths", + } + } + + const fn deleted_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown deleted paths", + Self::Workflow => "git workflow deleted paths", + } + } +} + +fn admit_paths<'a>( + repository_root: &Path, + paths: impl Iterator, + kind: CorpusKind, +) -> Result, DocumentationError> { + paths + .filter_map(|path| match admit_path(repository_root, path, kind) { + Ok(Some(path)) => Some(Ok(path)), + Ok(None) => None, + Err(error) => Some(Err(error)), + }) + .collect() +} + +fn admit_path( + repository_root: &Path, + path: &GitPath, + kind: CorpusKind, +) -> Result, DocumentationError> { + let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { + DocumentationError::PathEncoding { + corpus: kind.label(), + source, + } + })?; + let relative = posix_relative_path(&text).map_err(|_| DocumentationError::InvalidPath { + corpus: kind.label(), + path: text.clone(), + })?; + let metadata = match fs::symlink_metadata(repository_root.join(relative)) { + Ok(metadata) => metadata, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(source) => { + return Err(DocumentationError::Inspect { + corpus: kind.label(), + path: text, + source, + }); + } + }; + if metadata.file_type().is_file() { + Ok(Some(text)) + } else { + Err(DocumentationError::NonRegular { + corpus: kind.label(), + path: text, + }) + } +} + +#[cfg(test)] +#[path = "corpus/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs new file mode 100644 index 0000000..b606c0c --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -0,0 +1,178 @@ +//! This module owns documentation source-corpus regression evidence. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use super::{CorpusKind, SourceCorpus, admit_path}; +use crate::documentation_integrity::error::DocumentationError; +use crate::git_inventory::GitPath; +use crate::test_directory::TestDirectory; + +#[test] +fn markdown_corpus_is_the_sorted_present_repository_set() -> Result<(), Box> +{ + let directory = TestDirectory::create("documentation-markdown-corpus")?; + let root = directory.path(); + run_git(root, &["init", "--quiet"])?; + write(root, ".gitignore", "/target/\n")?; + write(root, "zulu.md", "# Zulu\n")?; + write(root, "alpha.md", "# Alpha\n")?; + write(root, "deleted.md", "# Deleted\n")?; + write(root, "target/generated.md", "# Generated\n")?; + run_git(root, &["add", ".gitignore", "zulu.md", "deleted.md"])?; + fs::remove_file(root.join("deleted.md"))?; + + let corpus = SourceCorpus::markdown(root)?; + + assert_eq!(corpus.paths(), ["alpha.md", "zulu.md"]); + directory.close()?; + Ok(()) +} + +#[test] +fn workflow_corpus_is_the_sorted_present_repository_set() -> Result<(), Box> +{ + let directory = TestDirectory::create("documentation-workflow-corpus")?; + let root = directory.path(); + run_git(root, &["init", "--quiet"])?; + write(root, ".gitignore", "/.github/workflows/generated.yml\n")?; + write(root, ".github/workflows/zulu.yml", "name: Zulu\n")?; + write(root, ".github/workflows/alpha.yaml", "name: Alpha\n")?; + write(root, ".github/workflows/deleted.yml", "name: Deleted\n")?; + write(root, ".github/workflows/generated.yml", "name: Generated\n")?; + run_git( + root, + &[ + "add", + ".gitignore", + ".github/workflows/zulu.yml", + ".github/workflows/deleted.yml", + ], + )?; + fs::remove_file(root.join(".github/workflows/deleted.yml"))?; + + let corpus = SourceCorpus::workflow(root)?; + + assert_eq!( + corpus.paths(), + [".github/workflows/alpha.yaml", ".github/workflows/zulu.yml"] + ); + directory.close()?; + Ok(()) +} + +#[test] +fn repository_configured_global_ignores_cannot_change_the_corpus() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-global-ignore")?; + let root = directory.path(); + run_git(root, &["init", "--quiet"])?; + write(root, "tracked.md", "# Tracked\n")?; + write(root, "new.md", "# New\n")?; + write(root, "global-ignore", "*.md\n")?; + run_git(root, &["add", "tracked.md"])?; + let global_ignore = root.join("global-ignore"); + let global_ignore = global_ignore + .to_str() + .ok_or("test path is not valid Unicode")?; + run_git(root, &["config", "core.excludesFile", global_ignore])?; + + let corpus = SourceCorpus::markdown(root)?; + + assert_eq!(corpus.paths(), ["new.md", "tracked.md"]); + directory.close()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn symlinked_markdown_is_refused() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let directory = TestDirectory::create("documentation-markdown-symlink")?; + let root = directory.path(); + run_git(root, &["init", "--quiet"])?; + write(root, "target.txt", "target\n")?; + symlink("target.txt", root.join("linked.md"))?; + + let result = SourceCorpus::markdown(root); + + assert!(matches!( + result, + Err(DocumentationError::NonRegular { + corpus: "Markdown", + ref path, + }) if path == "linked.md" + )); + directory.close()?; + Ok(()) +} + +#[cfg(unix)] +#[test] +fn fifo_workflow_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-workflow-fifo")?; + let root = directory.path(); + run_git(root, &["init", "--quiet"])?; + write(root, ".github/workflows/blocking.yml", "name: Blocking\n")?; + run_git(root, &["add", ".github/workflows/blocking.yml"])?; + let fifo = root.join(".github/workflows/blocking.yml"); + fs::remove_file(&fifo)?; + let status = Command::new("mkfifo").arg(&fifo).status()?; + if !status.success() { + return Err("mkfifo fixture command failed".into()); + } + + let result = SourceCorpus::workflow(root); + + assert!(matches!( + result, + Err(DocumentationError::NonRegular { + corpus: "GitHub Actions workflow", + ref path, + }) if path == ".github/workflows/blocking.yml" + )); + directory.close()?; + Ok(()) +} + +#[test] +fn non_utf8_markdown_path_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-markdown-non-utf8")?; + let root = directory.path(); + let path = GitPath::new(b"bad\xff.md".to_vec()); + + let result = admit_path(root, &path, CorpusKind::Markdown); + + assert!(matches!( + result, + Err(DocumentationError::PathEncoding { + corpus: "Markdown", + .. + }) + )); + directory.close()?; + Ok(()) +} + +fn run_git(root: &Path, arguments: &[&str]) -> Result<(), Box> { + let output = Command::new("git") + .args(arguments) + .current_dir(root) + .output()?; + if output.status.success() { + Ok(()) + } else { + Err(format!("git fixture command failed: {arguments:?}").into()) + } +} + +fn write(root: &Path, relative: &str, contents: &str) -> Result<(), Box> { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, contents)?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs new file mode 100644 index 0000000..d796c48 --- /dev/null +++ b/xtask/src/documentation_integrity/error.rs @@ -0,0 +1,81 @@ +//! This module owns typed documentation-integrity failures. + +use std::error::Error; +use std::fmt; +use std::io; +use std::string::FromUtf8Error; + +use crate::diagnostic::escaped_controls; +use crate::git_inventory::GitInventoryError; + +pub(super) enum DocumentationError { + EmptyCorpus(&'static str), + GitInventory(GitInventoryError), + Inspect { + corpus: &'static str, + path: String, + source: io::Error, + }, + InvalidPath { + corpus: &'static str, + path: String, + }, + NonRegular { + corpus: &'static str, + path: String, + }, + PathEncoding { + corpus: &'static str, + source: FromUtf8Error, + }, +} + +impl fmt::Debug for DocumentationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for DocumentationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), + Self::GitInventory(error) => write!(formatter, "{error}"), + Self::Inspect { corpus, path, .. } => { + write!(formatter, "cannot inspect {corpus} source `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::InvalidPath { corpus, path } => { + write!(formatter, "{corpus} corpus contains an unsafe path `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::NonRegular { corpus, path } => { + write!(formatter, "{corpus} source is not a regular file: `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::PathEncoding { corpus, .. } => { + write!(formatter, "{corpus} corpus contains a non-UTF-8 path") + } + } + } +} + +impl Error for DocumentationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::GitInventory(error) => Some(error), + Self::Inspect { source, .. } => Some(source), + Self::PathEncoding { source, .. } => Some(source), + Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } => None, + } + } +} + +impl From for DocumentationError { + fn from(error: GitInventoryError) -> Self { + Self::GitInventory(error) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 3afc2d8..aedf4e7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -9,6 +9,8 @@ )] mod benchmark_baseline; mod diagnostic; +#[cfg(test)] +mod documentation_integrity; #[allow( clippy::redundant_pub_crate, reason = "the command and task-error boundaries are sibling consumers" From 889207a46c5abc3143577cdcf737fe52d184f2b9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 11:55:20 -0700 Subject: [PATCH 003/113] Extract bounded process execution --- .../process.rs => bounded_process.rs} | 76 ++++++++++------ xtask/src/bounded_process/error.rs | 89 +++++++++++++++++++ xtask/src/fuzz_campaign.rs | 1 - xtask/src/fuzz_campaign/execution.rs | 15 +++- xtask/src/fuzz_campaign/execution/error.rs | 2 +- xtask/src/fuzz_campaign/execution/tests.rs | 8 +- xtask/src/fuzz_campaign/process/error.rs | 72 --------------- xtask/src/fuzz_campaign/target.rs | 6 +- xtask/src/fuzz_campaign/target/error.rs | 2 +- xtask/src/main.rs | 5 ++ 10 files changed, 164 insertions(+), 112 deletions(-) rename xtask/src/{fuzz_campaign/process.rs => bounded_process.rs} (68%) create mode 100644 xtask/src/bounded_process/error.rs delete mode 100644 xtask/src/fuzz_campaign/process/error.rs diff --git a/xtask/src/fuzz_campaign/process.rs b/xtask/src/bounded_process.rs similarity index 68% rename from xtask/src/fuzz_campaign/process.rs rename to xtask/src/bounded_process.rs index aebfd32..dc35848 100644 --- a/xtask/src/fuzz_campaign/process.rs +++ b/xtask/src/bounded_process.rs @@ -1,4 +1,4 @@ -//! This module owns bounded cargo-fuzz child-process collection. +//! This module owns bounded external child-process collection. mod error; @@ -7,20 +7,24 @@ use std::process::{Child, Command, ExitStatus, Stdio}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; -pub(super) use error::ProcessError; +pub(crate) use error::ProcessError; use crate::process_output::{BoundedBytes, bounded_bytes}; const OUTPUT_LIMIT: usize = 1_048_576; -pub(super) struct ProcessOutput { - pub(super) succeeded: bool, - pub(super) stdout: Vec, - pub(super) stderr: Vec, +pub(crate) struct ProcessOutput { + pub(crate) succeeded: bool, + pub(crate) stdout: Vec, + pub(crate) stderr: Vec, } -pub(super) fn status(command: &mut Command) -> Result { +pub(crate) fn status( + program: &'static str, + command: &mut Command, +) -> Result { let status = command.status().map_err(|source| ProcessError::Io { + program, action: "wait for", source, })?; @@ -31,47 +35,49 @@ pub(super) fn status(command: &mut Command) -> Result, ) -> Result { command.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = command.spawn().map_err(|source| ProcessError::Io { + program, action: "spawn", source, })?; - let stdout = child - .stdout - .take() - .ok_or(ProcessError::MissingStream("stdout")); - let stderr = child - .stderr - .take() - .ok_or(ProcessError::MissingStream("stderr")); + let stdout = child.stdout.take().ok_or(ProcessError::MissingStream { + program, + stream: "stdout", + }); + let stderr = child.stderr.take().ok_or(ProcessError::MissingStream { + program, + stream: "stderr", + }); let (stdout, stderr) = match (stdout, stderr) { (Ok(stdout), Ok(stderr)) => (stdout, stderr), (Err(error), _) | (_, Err(error)) => return Err(cleanup(&mut child, error)), }; - let stdout_reader = match start_reader("stdout", stdout) { + let stdout_reader = match start_reader(program, "stdout", stdout) { Ok(reader) => reader, Err(error) => return Err(cleanup(&mut child, error)), }; - let stderr_reader = match start_reader("stderr", stderr) { + let stderr_reader = match start_reader(program, "stderr", stderr) { Ok(reader) => reader, Err(error) => { let error = cleanup(&mut child, error); - drop(join_reader("stdout", stdout_reader)); + drop(join_reader(program, "stdout", stdout_reader)); return Err(error); } }; - let status = wait_for_child(&mut child, deadline); - let stdout = join_reader("stdout", stdout_reader); - let stderr = join_reader("stderr", stderr_reader); + let status = wait_for_child(program, &mut child, deadline); + let stdout = join_reader(program, "stdout", stdout_reader); + let stderr = join_reader(program, "stderr", stderr_reader); let status = status?; let stdout = stdout?; let stderr = stderr?; - refuse_exceeded("stdout", &stdout)?; - refuse_exceeded("stderr", &stderr)?; + refuse_exceeded(program, "stdout", &stdout)?; + refuse_exceeded(program, "stderr", &stderr)?; Ok(ProcessOutput { succeeded: status.success(), stdout: stdout.bytes, @@ -80,6 +86,7 @@ pub(super) fn capture( } fn wait_for_child( + program: &'static str, child: &mut Child, deadline: Option, ) -> Result { @@ -89,6 +96,7 @@ fn wait_for_child( Err(source) => Err(cleanup( child, ProcessError::Io { + program, action: "wait", source, }, @@ -96,7 +104,7 @@ fn wait_for_child( }; }; let Some(expires) = Instant::now().checked_add(duration) else { - return Err(cleanup(child, ProcessError::Timeout(duration))); + return Err(cleanup(child, ProcessError::Timeout { program, duration })); }; loop { match child.try_wait() { @@ -104,6 +112,7 @@ fn wait_for_child( return Err(cleanup( child, ProcessError::Io { + program, action: "poll", source, }, @@ -111,7 +120,7 @@ fn wait_for_child( } Ok(Some(status)) => return Ok(status), Ok(None) if Instant::now() >= expires => { - return Err(cleanup(child, ProcessError::Timeout(duration))); + return Err(cleanup(child, ProcessError::Timeout { program, duration })); } Ok(None) => thread::sleep(Duration::from_millis(10)), } @@ -119,34 +128,43 @@ fn wait_for_child( } fn start_reader( + program: &'static str, stream: &'static str, reader: impl io::Read + Send + 'static, ) -> Result>, ProcessError> { thread::Builder::new() - .name(format!("fuzz-{stream}-reader")) + .name(format!("xtask-{stream}-reader")) .spawn(move || bounded_bytes(reader, OUTPUT_LIMIT)) .map_err(|source| ProcessError::Io { + program, action: "start output reader", source, }) } fn join_reader( + program: &'static str, stream: &'static str, worker: JoinHandle>, ) -> Result { worker .join() - .map_err(|_panic| ProcessError::ReaderPanic(stream))? + .map_err(|_panic| ProcessError::ReaderPanic { program, stream })? .map_err(|source| ProcessError::Io { + program, action: "read child output", source, }) } -const fn refuse_exceeded(stream: &'static str, output: &BoundedBytes) -> Result<(), ProcessError> { +const fn refuse_exceeded( + program: &'static str, + stream: &'static str, + output: &BoundedBytes, +) -> Result<(), ProcessError> { if output.exceeded { Err(ProcessError::OutputLimit { + program, stream, maximum: OUTPUT_LIMIT, }) diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs new file mode 100644 index 0000000..3f89730 --- /dev/null +++ b/xtask/src/bounded_process/error.rs @@ -0,0 +1,89 @@ +//! This module owns typed bounded-process failures. + +use std::error::Error; +use std::fmt; +use std::io; +use std::time::Duration; + +pub(crate) enum ProcessError { + Cleanup { + primary: Box, + action: &'static str, + source: io::Error, + }, + Io { + program: &'static str, + action: &'static str, + source: io::Error, + }, + MissingStream { + program: &'static str, + stream: &'static str, + }, + OutputLimit { + program: &'static str, + stream: &'static str, + maximum: usize, + }, + ReaderPanic { + program: &'static str, + stream: &'static str, + }, + Timeout { + program: &'static str, + duration: Duration, + }, +} + +impl fmt::Debug for ProcessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, formatter) + } +} + +impl fmt::Display for ProcessError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Cleanup { + primary, action, .. + } => write!( + formatter, + "{primary}; additionally failed to {action} child process" + ), + Self::Io { + program, action, .. + } => write!(formatter, "cannot {action} {program} process"), + Self::MissingStream { program, stream } => { + write!(formatter, "{program} {stream} pipe is unavailable") + } + Self::OutputLimit { + program, + stream, + maximum, + } => write!( + formatter, + "{program} {stream} exceeds the {maximum}-byte bound" + ), + Self::ReaderPanic { program, stream } => { + write!(formatter, "{program} {stream} reader panicked") + } + Self::Timeout { program, duration } => write!( + formatter, + "{program} process exceeded its {}-second deadline", + duration.as_secs() + ), + } + } +} + +impl Error for ProcessError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), + Self::MissingStream { .. } + | Self::OutputLimit { .. } + | Self::ReaderPanic { .. } + | Self::Timeout { .. } => None, + } + } +} diff --git a/xtask/src/fuzz_campaign.rs b/xtask/src/fuzz_campaign.rs index 87c7e38..9c1bcde 100644 --- a/xtask/src/fuzz_campaign.rs +++ b/xtask/src/fuzz_campaign.rs @@ -5,7 +5,6 @@ mod corpus; mod error; mod execution; mod policy; -mod process; mod profile; mod target; #[cfg(test)] diff --git a/xtask/src/fuzz_campaign/execution.rs b/xtask/src/fuzz_campaign/execution.rs index fa0bd69..ff3abdc 100644 --- a/xtask/src/fuzz_campaign/execution.rs +++ b/xtask/src/fuzz_campaign/execution.rs @@ -12,7 +12,9 @@ pub(crate) use error::ExecutionError; use error::{TargetFailure, TargetFailureReason}; use super::command::{CommandPlan, OutputMode}; -use super::process::{self, ProcessError, ProcessOutput}; +use crate::bounded_process::{self, ProcessError, ProcessOutput}; + +const CARGO_FUZZ_PROCESS: &str = "cargo-fuzz"; trait CommandRunner { fn execute( @@ -34,11 +36,12 @@ impl CommandRunner for SystemRunner { command.args(plan.arguments()).current_dir(repository_root); match plan.output_mode() { OutputMode::Capture => { - let output = process::capture(&mut command, plan.deadline())?; + let output = + bounded_process::capture(CARGO_FUZZ_PROCESS, &mut command, plan.deadline())?; replay(&output)?; Ok(output) } - OutputMode::Inherit => process::status(&mut command), + OutputMode::Inherit => bounded_process::status(CARGO_FUZZ_PROCESS, &mut command), } } } @@ -72,7 +75,11 @@ fn write_output( writer .write_all(bytes) .and_then(|()| writer.flush()) - .map_err(|source| ProcessError::Io { action, source }) + .map_err(|source| ProcessError::Io { + program: CARGO_FUZZ_PROCESS, + action, + source, + }) } fn execute_all( diff --git a/xtask/src/fuzz_campaign/execution/error.rs b/xtask/src/fuzz_campaign/execution/error.rs index 3ccad36..2311fdf 100644 --- a/xtask/src/fuzz_campaign/execution/error.rs +++ b/xtask/src/fuzz_campaign/execution/error.rs @@ -3,7 +3,7 @@ use std::error::Error; use std::fmt; -use crate::fuzz_campaign::process::ProcessError; +use crate::bounded_process::ProcessError; pub(crate) struct ExecutionError { pub(super) operation: &'static str, diff --git a/xtask/src/fuzz_campaign/execution/tests.rs b/xtask/src/fuzz_campaign/execution/tests.rs index 34ce815..0186a18 100644 --- a/xtask/src/fuzz_campaign/execution/tests.rs +++ b/xtask/src/fuzz_campaign/execution/tests.rs @@ -4,9 +4,9 @@ use std::io; use std::path::Path; use super::{CommandRunner, execute_all}; +use crate::bounded_process::{ProcessError, ProcessOutput}; use crate::fuzz_campaign::command::{CampaignOperation, CommandPlan}; use crate::fuzz_campaign::policy::CampaignPolicy; -use crate::fuzz_campaign::process::{ProcessError, ProcessOutput}; use crate::fuzz_campaign::profile::CampaignProfile; use crate::fuzz_campaign::target::FuzzTarget; @@ -103,6 +103,7 @@ impl CommandRunner for RefusingRunner { _plan: &CommandPlan, ) -> Result { Err(ProcessError::Io { + program: "cargo-fuzz", action: "spawn", source: io::Error::other("scripted refusal"), }) @@ -126,7 +127,10 @@ impl CommandRunner for ScriptedRunner { ) -> Result { self.observed.push(plan.target().as_str().to_owned()); let Some(output) = self.outcomes.pop_front() else { - return Err(ProcessError::MissingStream("scripted outcome")); + return Err(ProcessError::MissingStream { + program: "cargo-fuzz", + stream: "scripted outcome", + }); }; Ok(output) } diff --git a/xtask/src/fuzz_campaign/process/error.rs b/xtask/src/fuzz_campaign/process/error.rs deleted file mode 100644 index 2308a69..0000000 --- a/xtask/src/fuzz_campaign/process/error.rs +++ /dev/null @@ -1,72 +0,0 @@ -//! This module owns typed cargo-fuzz process failures. - -use std::error::Error; -use std::fmt; -use std::io; -use std::time::Duration; - -pub(crate) enum ProcessError { - Cleanup { - primary: Box, - action: &'static str, - source: io::Error, - }, - Io { - action: &'static str, - source: io::Error, - }, - MissingStream(&'static str), - OutputLimit { - stream: &'static str, - maximum: usize, - }, - ReaderPanic(&'static str), - Timeout(Duration), -} - -impl fmt::Debug for ProcessError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, formatter) - } -} - -impl fmt::Display for ProcessError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Cleanup { - primary, action, .. - } => write!( - formatter, - "{primary}; additionally failed to {action} child process" - ), - Self::Io { action, .. } => write!(formatter, "cannot {action} cargo-fuzz process"), - Self::MissingStream(stream) => { - write!(formatter, "cargo-fuzz {stream} pipe is unavailable") - } - Self::OutputLimit { stream, maximum } => { - write!( - formatter, - "cargo-fuzz {stream} exceeds the {maximum}-byte bound" - ) - } - Self::ReaderPanic(stream) => write!(formatter, "cargo-fuzz {stream} reader panicked"), - Self::Timeout(duration) => write!( - formatter, - "cargo-fuzz process exceeded its {}-second deadline", - duration.as_secs() - ), - } - } -} - -impl Error for ProcessError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), - Self::MissingStream(_) - | Self::OutputLimit { .. } - | Self::ReaderPanic(_) - | Self::Timeout(_) => None, - } - } -} diff --git a/xtask/src/fuzz_campaign/target.rs b/xtask/src/fuzz_campaign/target.rs index c3f5837..ff92e67 100644 --- a/xtask/src/fuzz_campaign/target.rs +++ b/xtask/src/fuzz_campaign/target.rs @@ -13,7 +13,9 @@ use std::process::Command; pub(super) use error::TargetError; use super::policy::CampaignPolicy; -use super::process; +use crate::bounded_process; + +const CARGO_FUZZ_PROCESS: &str = "cargo-fuzz"; #[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] pub(super) struct FuzzTarget(String); @@ -48,7 +50,7 @@ pub(super) fn registered( .arg(format!("+{}", policy.toolchain())) .args(["fuzz", "list"]) .current_dir(repository_root); - let output = process::capture(&mut command, None)?; + let output = bounded_process::capture(CARGO_FUZZ_PROCESS, &mut command, None)?; if !output.succeeded { return Err(TargetError::ListFailed { stdout: output.stdout, diff --git a/xtask/src/fuzz_campaign/target/error.rs b/xtask/src/fuzz_campaign/target/error.rs index 8c7c74f..7ea517c 100644 --- a/xtask/src/fuzz_campaign/target/error.rs +++ b/xtask/src/fuzz_campaign/target/error.rs @@ -5,8 +5,8 @@ use std::fmt; use std::io; use std::path::PathBuf; +use crate::bounded_process::ProcessError; use crate::diagnostic::{escaped_controls, escaped_path}; -use crate::fuzz_campaign::process::ProcessError; pub(crate) enum TargetError { Disagreement { diff --git a/xtask/src/main.rs b/xtask/src/main.rs index aedf4e7..347b347 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -8,6 +8,11 @@ reason = "the command dispatcher owns this private repository task" )] mod benchmark_baseline; +#[allow( + clippy::redundant_pub_crate, + reason = "bounded process execution is shared by sibling repository tasks" +)] +mod bounded_process; mod diagnostic; #[cfg(test)] mod documentation_integrity; From 69fa99ecb2d42e0ddf3ab39aa9ed6c2ecbf1d70c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 11:58:56 -0700 Subject: [PATCH 004/113] Add bounded documentation tool policy --- xtask/src/bounded_process.rs | 4 + xtask/src/bounded_process/tests.rs | 47 ++++++++++++ xtask/src/documentation_integrity.rs | 1 + xtask/src/documentation_integrity/error.rs | 18 ++++- xtask/src/documentation_integrity/tool.rs | 76 +++++++++++++++++++ .../src/documentation_integrity/tool/tests.rs | 63 +++++++++++++++ 6 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 xtask/src/bounded_process/tests.rs create mode 100644 xtask/src/documentation_integrity/tool.rs create mode 100644 xtask/src/documentation_integrity/tool/tests.rs diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index dc35848..4c439ec 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -194,3 +194,7 @@ fn cleanup(child: &mut std::process::Child, primary: ProcessError) -> ProcessErr } primary } + +#[cfg(test)] +#[path = "bounded_process/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs new file mode 100644 index 0000000..4a419ae --- /dev/null +++ b/xtask/src/bounded_process/tests.rs @@ -0,0 +1,47 @@ +//! This module owns bounded child-process regression evidence. + +use std::env; +use std::io::{self, Write}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use super::{ProcessError, capture}; + +const OUTPUT_CHILD: &str = "KEEP_XTASK_BOUNDED_OUTPUT_CHILD"; + +#[test] +fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box> +{ + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::tests::process_child_writes_excess_output", + ]) + .env(OUTPUT_CHILD, "1") + .stdin(Stdio::null()); + + let result = capture("test process", &mut command, Some(Duration::from_secs(5))); + + assert!(matches!( + result, + Err(ProcessError::OutputLimit { + program: "test process", + stream: "stdout", + maximum: 1_048_576, + }) + )); + Ok(()) +} + +#[test] +fn process_child_writes_excess_output() -> Result<(), io::Error> { + if env::var_os(OUTPUT_CHILD).is_none() { + return Ok(()); + } + let bytes = vec![b'x'; 1_048_577]; + let mut output = io::stdout().lock(); + output.write_all(&bytes)?; + output.flush() +} diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 802d9c3..6c875c5 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -2,3 +2,4 @@ mod corpus; mod error; +mod tool; diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index d796c48..689aab8 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -28,6 +28,11 @@ pub(super) enum DocumentationError { corpus: &'static str, source: FromUtf8Error, }, + VersionMismatch { + program: &'static str, + expected: &'static str, + observed: String, + }, } impl fmt::Debug for DocumentationError { @@ -59,6 +64,14 @@ impl fmt::Display for DocumentationError { Self::PathEncoding { corpus, .. } => { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } + Self::VersionMismatch { + program, + expected, + observed, + } => write!( + formatter, + "{program} version mismatch: expected {expected:?}, observed {observed:?}" + ), } } } @@ -69,7 +82,10 @@ impl Error for DocumentationError { Self::GitInventory(error) => Some(error), Self::Inspect { source, .. } => Some(source), Self::PathEncoding { source, .. } => Some(source), - Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } => None, + Self::EmptyCorpus(_) + | Self::InvalidPath { .. } + | Self::NonRegular { .. } + | Self::VersionMismatch { .. } => None, } } } diff --git a/xtask/src/documentation_integrity/tool.rs b/xtask/src/documentation_integrity/tool.rs new file mode 100644 index 0000000..9d40efb --- /dev/null +++ b/xtask/src/documentation_integrity/tool.rs @@ -0,0 +1,76 @@ +//! This module owns admitted documentation-tool versions and arguments. + +use super::error::DocumentationError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum DocumentationTool { + Actionlint, + Lychee, + Markdownlint, +} + +impl DocumentationTool { + pub(super) const fn program(self) -> &'static str { + match self { + Self::Actionlint => "actionlint", + Self::Lychee => "lychee", + Self::Markdownlint => "markdownlint-cli2", + } + } + + pub(super) const fn install_version(self) -> &'static str { + match self { + Self::Actionlint => "1.7.12", + Self::Lychee => "0.21.0", + Self::Markdownlint => "0.23.2", + } + } + + pub(super) const fn expected_version(self) -> &'static str { + match self { + Self::Actionlint => "1.7.12", + Self::Lychee => "lychee 0.21.0", + Self::Markdownlint => "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + } + } + + pub(super) const fn version_arguments(self) -> &'static [&'static str] { + match self { + Self::Actionlint => &["-version"], + Self::Lychee => &["--version"], + Self::Markdownlint => &["--no-globs", "--version"], + } + } + + pub(super) const fn check_prefix(self) -> &'static [&'static str] { + match self { + Self::Actionlint => &["-shellcheck=", "-pyflakes="], + Self::Lychee => &[ + "--offline", + "--include-fragments", + "--no-progress", + "--format", + "detailed", + "--", + ], + Self::Markdownlint => &["--no-globs", "--"], + } + } + + pub(super) fn admit_version(self, observed: &str) -> Result<(), DocumentationError> { + let expected = self.expected_version(); + if observed == expected { + Ok(()) + } else { + Err(DocumentationError::VersionMismatch { + program: self.program(), + expected, + observed: observed.to_owned(), + }) + } + } +} + +#[cfg(test)] +#[path = "tool/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/tool/tests.rs b/xtask/src/documentation_integrity/tool/tests.rs new file mode 100644 index 0000000..7e8a739 --- /dev/null +++ b/xtask/src/documentation_integrity/tool/tests.rs @@ -0,0 +1,63 @@ +//! This module owns documentation-tool policy regression evidence. + +use super::DocumentationTool; +use crate::documentation_integrity::error::DocumentationError; + +#[test] +fn every_unreviewed_tool_version_is_refused() { + for tool in tools() { + assert!(matches!( + tool.admit_version("999.0.0"), + Err(DocumentationError::VersionMismatch { + program, + expected, + ref observed, + }) if program == tool.program() + && expected == tool.expected_version() + && observed == "999.0.0" + )); + } +} + +#[test] +fn every_reviewed_tool_version_is_admitted_exactly() { + for tool in tools() { + assert!(tool.admit_version(tool.expected_version()).is_ok()); + assert!(!tool.install_version().is_empty()); + } +} + +#[test] +fn tool_arguments_preserve_the_reviewed_execution_boundary() { + assert_eq!( + DocumentationTool::Markdownlint.version_arguments(), + ["--no-globs", "--version"] + ); + assert_eq!( + DocumentationTool::Markdownlint.check_prefix(), + ["--no-globs", "--"] + ); + assert_eq!( + DocumentationTool::Lychee.check_prefix(), + [ + "--offline", + "--include-fragments", + "--no-progress", + "--format", + "detailed", + "--", + ] + ); + assert_eq!( + DocumentationTool::Actionlint.check_prefix(), + ["-shellcheck=", "-pyflakes="] + ); +} + +const fn tools() -> [DocumentationTool; 3] { + [ + DocumentationTool::Markdownlint, + DocumentationTool::Lychee, + DocumentationTool::Actionlint, + ] +} From c7554d843efe1bebd09361b3605d176d6cbda52f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:03:51 -0700 Subject: [PATCH 005/113] Extract repository file admission --- xtask/src/main.rs | 5 +++ .../source_file.rs => repository_file.rs} | 42 +++++++++---------- xtask/src/source_structure.rs | 21 +++++----- xtask/src/source_structure/tests.rs | 30 ++++++++----- xtask/tests/source_policy_contract.rs | 8 ++-- 5 files changed, 59 insertions(+), 47 deletions(-) rename xtask/src/{source_structure/source_file.rs => repository_file.rs} (66%) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 347b347..e16766b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -46,6 +46,11 @@ mod process_output; reason = "the command and task-error boundaries are sibling consumers" )] mod protocol_conformance; +#[allow( + clippy::redundant_pub_crate, + reason = "repository file admission is shared by sibling repository tasks" +)] +mod repository_file; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" diff --git a/xtask/src/source_structure/source_file.rs b/xtask/src/repository_file.rs similarity index 66% rename from xtask/src/source_structure/source_file.rs rename to xtask/src/repository_file.rs index bdd94c0..b069669 100644 --- a/xtask/src/source_structure/source_file.rs +++ b/xtask/src/repository_file.rs @@ -1,9 +1,9 @@ -//! This module owns capability-relative, no-follow source-file admission. +//! This module owns capability-relative, no-follow repository-file admission. //! -//! The repository source verifier is intentionally supported only on Unix hosts. -//! It binds an opened source root to Unix device and inode identity so that path -//! replacement cannot silently redirect a scan. Supporting another host requires -//! an equivalent stable directory-identity contract before enabling this task. +//! Repository tasks are intentionally supported only on Unix hosts. This module +//! binds an opened repository root to Unix device and inode identity so that path +//! replacement cannot silently redirect a read. Supporting another host requires +//! an equivalent stable directory-identity contract before enabling these tasks. use std::fs::File; use std::io; @@ -14,27 +14,27 @@ use cap_std::ambient_authority; use cap_std::fs::{Dir, OpenOptions}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum ReadAccessPolicy { +pub(crate) enum ReadAccessPolicy { Enabled, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum BlockingIoPolicy { +pub(crate) enum BlockingIoPolicy { Refuse, } #[derive(Clone, Copy)] -pub(super) struct SourceReadPolicy { +pub(crate) struct RepositoryReadPolicy { read_access: ReadAccessPolicy, blocking_io: BlockingIoPolicy, } -pub(super) const SOURCE_READ_POLICY: SourceReadPolicy = SourceReadPolicy { +pub(crate) const REPOSITORY_READ_POLICY: RepositoryReadPolicy = RepositoryReadPolicy { read_access: ReadAccessPolicy::Enabled, blocking_io: BlockingIoPolicy::Refuse, }; -pub(super) struct SourceRoot { +pub(crate) struct RepositoryRoot { directory: Dir, identity: DirectoryIdentity, path: PathBuf, @@ -46,13 +46,13 @@ struct DirectoryIdentity { inode: u64, } -pub(super) enum OpenSourceError { +pub(crate) enum OpenRepositoryFileError { Io(io::Error), NonRegular, } -impl SourceRoot { - pub(super) fn open(path: &Path) -> Result { +impl RepositoryRoot { + pub(crate) fn open(path: &Path) -> Result { let directory = Dir::open_ambient_dir(path, ambient_authority())?; let identity = DirectoryIdentity::from(&directory.dir_metadata()?); Ok(Self { @@ -62,32 +62,32 @@ impl SourceRoot { }) } - pub(super) fn display_path(&self, relative: &Path) -> PathBuf { + pub(crate) fn display_path(&self, relative: &Path) -> PathBuf { self.path.join(relative) } - pub(super) fn is_current_path(&self) -> Result { + pub(crate) fn is_current_path(&self) -> Result { let current = Dir::open_ambient_dir(&self.path, ambient_authority())?; let identity = DirectoryIdentity::from(¤t.dir_metadata()?); Ok(self.identity == identity) } - pub(super) fn open_file(&self, relative: &Path) -> Result { + pub(crate) fn open_file(&self, relative: &Path) -> Result { let file = self .directory - .open_with(relative, &SOURCE_READ_POLICY.options()) - .map_err(OpenSourceError::Io)? + .open_with(relative, &REPOSITORY_READ_POLICY.options()) + .map_err(OpenRepositoryFileError::Io)? .into_std(); - let metadata = file.metadata().map_err(OpenSourceError::Io)?; + let metadata = file.metadata().map_err(OpenRepositoryFileError::Io)?; if metadata.is_file() { Ok(file) } else { - Err(OpenSourceError::NonRegular) + Err(OpenRepositoryFileError::NonRegular) } } } -impl SourceReadPolicy { +impl RepositoryReadPolicy { #[cfg(test)] pub(super) const fn read_access(self) -> ReadAccessPolicy { self.read_access diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 8aa3ad5..579bace 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -2,16 +2,15 @@ mod repository_path; mod source_error; -mod source_file; use std::collections::BTreeSet; use std::io::{self, BufRead, BufReader}; use std::path::Path; use crate::git_inventory::{GitPath, paths as git_paths}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; -use source_file::{OpenSourceError, SourceRoot}; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; const SOURCE_SUFFIXES: [[u8; 2]; 3] = [*b"py", *b"rs", *b"sh"]; @@ -25,7 +24,7 @@ const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> { let source_root = - SourceRoot::open(repository_root).map_err(|source| SourceStructureError::Inspect { + RepositoryRoot::open(repository_root).map_err(|source| SourceStructureError::Inspect { path: repository_root.to_owned(), source, })?; @@ -43,7 +42,7 @@ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> } fn verify_source_root( - source_root: &SourceRoot, + source_root: &RepositoryRoot, repository_root: &Path, ) -> Result<(), SourceStructureError> { match source_root.is_current_path() { @@ -94,7 +93,7 @@ fn admit_source_path(path: &GitPath) -> Result, ) -> Result, SourceStructureError> { let mut violations = Vec::new(); @@ -108,24 +107,24 @@ fn source_violations( } fn source_line_count( - source_root: &SourceRoot, + source_root: &RepositoryRoot, relative: &RepositoryPath, ) -> Result { - source_line_count_with(source_root, relative, SourceRoot::open_file) + source_line_count_with(source_root, relative, RepositoryRoot::open_file) } fn source_line_count_with( - source_root: &SourceRoot, + source_root: &RepositoryRoot, relative: &RepositoryPath, - open_source: impl FnOnce(&SourceRoot, &Path) -> Result, + open_source: impl FnOnce(&RepositoryRoot, &Path) -> Result, ) -> Result { let path = source_root.display_path(relative.as_path()); let file = open_source(source_root, relative.as_path()).map_err(|error| match error { - OpenSourceError::Io(source) => SourceStructureError::Inspect { + OpenRepositoryFileError::Io(source) => SourceStructureError::Inspect { path: path.clone(), source, }, - OpenSourceError::NonRegular => SourceStructureError::NonRegular(path.clone()), + OpenRepositoryFileError::NonRegular => SourceStructureError::NonRegular(path.clone()), })?; line_count(BufReader::new(file)) .map_err(|source| SourceStructureError::Inspect { path, source }) diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index eef2886..01e8721 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -137,10 +137,16 @@ fn git_diagnostics_cannot_inject_terminal_control_lines() { #[test] fn source_read_policy_enables_reads_and_refuses_blocking_io() { - use super::source_file::{BlockingIoPolicy, ReadAccessPolicy, SOURCE_READ_POLICY}; + use crate::repository_file::{BlockingIoPolicy, REPOSITORY_READ_POLICY, ReadAccessPolicy}; - assert_eq!(SOURCE_READ_POLICY.read_access(), ReadAccessPolicy::Enabled); - assert_eq!(SOURCE_READ_POLICY.blocking_io(), BlockingIoPolicy::Refuse); + assert_eq!( + REPOSITORY_READ_POLICY.read_access(), + ReadAccessPolicy::Enabled + ); + assert_eq!( + REPOSITORY_READ_POLICY.blocking_io(), + BlockingIoPolicy::Refuse + ); } #[cfg(unix)] @@ -148,8 +154,9 @@ fn source_read_policy_enables_reads_and_refuses_blocking_io() { fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { use std::fs; + use crate::repository_file::RepositoryRoot; + use super::repository_path::RepositoryPath; - use super::source_file::SourceRoot; use super::source_line_count; let directory = TestDirectory::create("source-root")?; @@ -157,7 +164,7 @@ fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box Result<(), Box Result<(), Box> { use std::fs; - use super::source_file::SourceRoot; + use crate::repository_file::RepositoryRoot; let directory = TestDirectory::create("source-identity")?; let root = directory.path().join("repository"); let retained_root = directory.path().join("retained"); fs::create_dir(&root)?; - let source_root = SourceRoot::open(&root)?; + let source_root = RepositoryRoot::open(&root)?; fs::rename(&root, &retained_root)?; fs::create_dir(&root)?; @@ -202,8 +209,9 @@ fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructur use std::fs; use std::os::unix::fs::symlink; + use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; + use super::repository_path::RepositoryPath; - use super::source_file::SourceRoot; use super::source_line_count_with; let directory = TestDirectory::create("source-replacement").map_err(|source| { @@ -231,7 +239,7 @@ fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructur } })?; let source_root = - SourceRoot::open(&root).map_err(|source| super::SourceStructureError::Inspect { + RepositoryRoot::open(&root).map_err(|source| super::SourceStructureError::Inspect { path: root.clone(), source, })?; @@ -239,8 +247,8 @@ fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructur let result = source_line_count_with(&source_root, &relative, |source_root, relative| { let admitted = source_root.display_path(relative); - fs::rename(&admitted, &retained_path).map_err(super::OpenSourceError::Io)?; - symlink(&target_path, &admitted).map_err(super::OpenSourceError::Io)?; + fs::rename(&admitted, &retained_path).map_err(OpenRepositoryFileError::Io)?; + symlink(&target_path, &admitted).map_err(OpenRepositoryFileError::Io)?; source_root.open_file(relative) }); let refused = matches!( diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index c4faadc..5b488ba 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -1,7 +1,7 @@ //! Written-policy regression evidence for the executable source-size law. const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); -const SOURCE_FILE: &str = include_str!("../src/source_structure/source_file.rs"); +const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); #[test] @@ -16,7 +16,7 @@ fn written_source_limit_matches_the_executable_law() { } #[test] -fn source_root_identity_declares_its_unix_scope() { - assert!(SOURCE_FILE.contains("intentionally supported only on Unix hosts")); - assert!(SOURCE_FILE.contains("Unix device and inode identity")); +fn repository_file_admission_declares_its_unix_scope() { + assert!(REPOSITORY_FILE.contains("intentionally supported only on Unix hosts")); + assert!(REPOSITORY_FILE.contains("Unix device and inode identity")); } From b70071ce0265c3e778feb3a23f23d53a3d879c8a Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:13:06 -0700 Subject: [PATCH 006/113] Port documentation tool lock laws --- Cargo.lock | 74 +++++++- docs/dependencies/serde-json-1.0.151.md | 71 ++++++++ xtask/Cargo.toml | 10 +- xtask/src/documentation_integrity.rs | 2 + xtask/src/documentation_integrity/error.rs | 99 ++++++++++- .../documentation_integrity/node_toolchain.rs | 159 ++++++++++++++++++ .../node_toolchain/tests.rs | 119 +++++++++++++ .../repository_text.rs | 50 ++++++ .../repository_text/tests.rs | 52 ++++++ 9 files changed, 632 insertions(+), 4 deletions(-) create mode 100644 docs/dependencies/serde-json-1.0.151.md create mode 100644 xtask/src/documentation_integrity/node_toolchain.rs create mode 100644 xtask/src/documentation_integrity/node_toolchain/tests.rs create mode 100644 xtask/src/documentation_integrity/repository_text.rs create mode 100644 xtask/src/documentation_integrity/repository_text/tests.rs diff --git a/Cargo.lock b/Cargo.lock index adda953..aea746d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,7 +216,7 @@ checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -283,6 +283,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "keep" version = "0.0.0" @@ -329,6 +335,12 @@ dependencies = [ "digest", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "once_cell" version = "1.21.4" @@ -382,6 +394,48 @@ dependencies = [ "rustix", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" @@ -399,6 +453,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "typenum" version = "1.20.1" @@ -613,4 +678,11 @@ dependencies = [ "cap-fs-ext", "cap-std", "md-5", + "serde_json", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/docs/dependencies/serde-json-1.0.151.md b/docs/dependencies/serde-json-1.0.151.md new file mode 100644 index 0000000..5f91339 --- /dev/null +++ b/docs/dependencies/serde-json-1.0.151.md @@ -0,0 +1,71 @@ +# Dependency Admission: serde_json 1.0.151 + +- Status: Accepted for repository-task JSON admission only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [serde-rs/json](https://github.com/serde-rs/json) + +## Admitted use + +Keep admits the exactly pinned `serde_json` 1.0.151 package only behind the +`xtask` crate's `repository-tasks` feature. It parses the committed Node +documentation-tool manifest and lockfile so the Rust documentation integrity +task can validate their structure and reviewed dependency versions. + +The dependency is absent from Keep's published library graph, public API, +content identities, durable formats, and production behavior. No +dependency-owned type crosses out of the private repository-task adapter. + +## Why a dependency is needed + +JSON syntax includes escapes, Unicode, numbers, nested collections, and +duplicate representation details that do not belong in a local partial parser. +The documentation gate needs structural lookup, not canonical identity bytes. +Using a maintained parser keeps malformed input fail-closed without creating a +second JSON implementation inside Keep. + +The parsed values are never hashed, persisted, or admitted as Keep domain +types. The repository task reads fixed paths through the bounded, +capability-relative, no-follow file boundary before parsing them. + +## Features and resolved graph + +The direct dependency disables default features and enables only `std`. It is +optional and activated solely by `repository-tasks`. + +The active normal dependency graph introduced for this boundary consists of: + +- `itoa` 1.0.18; +- `memchr` 2.8.3; +- `serde_core` 1.0.229; and +- `zmij` 1.0.23. + +Cargo's all-target resolution also retains `serde` 1.0.229, `serde_derive` +1.0.229, and `syn` 3.0.3. Their procedural-macro dependencies were already +present in the workspace lockfile. + +## Safety, licensing, and compatibility + +`serde_json` declares the MIT OR Apache-2.0 license expression. Its manifest +declares Rust 1.71 as its minimum supported Rust version, below Keep's pinned +toolchain. + +Keep-owned code invokes only safe APIs. The parser and its transitive +dependencies may contain implementation details outside Keep's `unsafe_code` +lint boundary, so `cargo deny` and RustSec checks remain mandatory +point-in-time evidence. + +## Failure and recovery boundaries + +Malformed JSON, an oversized file, a non-UTF-8 file, a non-regular file, or a +repository-root replacement produces a typed refusal. The task never repairs, +rewrites, or substitutes repository data. Parsing has no durability or +recovery semantics. + +Keep can remove this dependency without changing public or durable behavior by +replacing it with an equally bounded parser that preserves the exact rejection +and structural-validation laws. + +Reopen this admission if the direct version, selected feature, resolved graph, +license, MSRV, repository-task-only boundary, or admitted JSON use changes. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index afb0695..1a51a63 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,7 +9,13 @@ publish = false [features] default = ["repository-tasks"] golden-protocol-fuzz = [] -repository-tasks = ["dep:blake3", "dep:cap-fs-ext", "dep:cap-std", "dep:md-5"] +repository-tasks = [ + "dep:blake3", + "dep:cap-fs-ext", + "dep:cap-std", + "dep:md-5", + "dep:serde_json", +] [dependencies] # Independent Golden File Worldline oracle recomputes BLAKE3 corpus witnesses. @@ -19,6 +25,8 @@ cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], cap-std = { version = "=4.0.2", default-features = false, optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } +# Typed JSON admission checks the committed documentation-tool lock graph. +serde_json = { version = "=1.0.151", default-features = false, features = ["std"], optional = true } [[bin]] name = "xtask" diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 6c875c5..a33e59c 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -2,4 +2,6 @@ mod corpus; mod error; +mod node_toolchain; +mod repository_text; mod tool; diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 689aab8..3994349 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -28,6 +28,38 @@ pub(super) enum DocumentationError { corpus: &'static str, source: FromUtf8Error, }, + RepositoryFileEncoding { + path: &'static str, + source: FromUtf8Error, + }, + RepositoryFileInspect { + path: &'static str, + source: io::Error, + }, + RepositoryFileNonRegular(&'static str), + RepositoryFileTooLarge { + path: &'static str, + maximum: u64, + }, + RepositoryContract { + path: &'static str, + requirement: &'static str, + }, + RepositoryContractAt { + path: &'static str, + subject: String, + requirement: &'static str, + }, + RepositoryJson { + path: &'static str, + source: serde_json::Error, + }, + RepositoryValue { + path: &'static str, + field: &'static str, + expected: &'static str, + observed: Option, + }, VersionMismatch { program: &'static str, expected: &'static str, @@ -64,6 +96,59 @@ impl fmt::Display for DocumentationError { Self::PathEncoding { corpus, .. } => { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } + Self::RepositoryFileEncoding { path, .. } => { + write!(formatter, "repository file `{path}` is not UTF-8") + } + Self::RepositoryFileInspect { path, .. } => { + write!(formatter, "cannot inspect repository file `{path}`") + } + Self::RepositoryFileNonRegular(path) => { + write!(formatter, "repository file is not regular: `{path}`") + } + Self::RepositoryFileTooLarge { path, maximum } => write!( + formatter, + "repository file `{path}` exceeds the {maximum}-byte bound" + ), + Self::RepositoryContract { path, requirement } => { + write!( + formatter, + "repository file `{path}` violates: {requirement}" + ) + } + Self::RepositoryContractAt { + path, + subject, + requirement, + } => { + write!( + formatter, + "repository file `{path}` violates {requirement} at `" + )?; + escaped_controls(formatter, subject)?; + formatter.write_str("`") + } + Self::RepositoryJson { path, .. } => { + write!(formatter, "repository file `{path}` is not valid JSON") + } + Self::RepositoryValue { + path, + field, + expected, + observed, + } => { + write!( + formatter, + "repository file `{path}` requires `{field}` to be {expected:?}; observed " + )?; + match observed { + Some(value) => { + formatter.write_str("\"")?; + escaped_controls(formatter, value)?; + formatter.write_str("\"") + } + None => formatter.write_str("missing"), + } + } Self::VersionMismatch { program, expected, @@ -80,11 +165,21 @@ impl Error for DocumentationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::GitInventory(error) => Some(error), - Self::Inspect { source, .. } => Some(source), - Self::PathEncoding { source, .. } => Some(source), + Self::Inspect { source, .. } | Self::RepositoryFileInspect { source, .. } => { + Some(source) + } + Self::PathEncoding { source, .. } | Self::RepositoryFileEncoding { source, .. } => { + Some(source) + } + Self::RepositoryJson { source, .. } => Some(source), Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } + | Self::RepositoryFileNonRegular(_) + | Self::RepositoryFileTooLarge { .. } + | Self::RepositoryContract { .. } + | Self::RepositoryContractAt { .. } + | Self::RepositoryValue { .. } | Self::VersionMismatch { .. } => None, } } diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs new file mode 100644 index 0000000..3944a1a --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -0,0 +1,159 @@ +//! This module owns the committed Node documentation-tool graph contract. + +use serde_json::{Map, Value}; + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text; + +const INSTALLER_PATH: &str = "scripts/install_documentation_tools.sh"; +const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; +const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; + +pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { + let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; + let lock = repository_text::read(repository_root, LOCK_PATH)?; + let installer = repository_text::read(repository_root, INSTALLER_PATH)?; + admit(&manifest, &lock, &installer) +} + +fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), DocumentationError> { + let manifest = parse(MANIFEST_PATH, manifest)?; + admit_manifest(&manifest)?; + let lock = parse(LOCK_PATH, lock)?; + admit_lock(&lock)?; + admit_installer(installer) +} + +fn admit_manifest(manifest: &Value) -> Result<(), DocumentationError> { + require( + manifest.get("overrides").is_none(), + MANIFEST_PATH, + "dependency overrides are absent", + ) +} + +fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { + require( + lock.get("lockfileVersion").and_then(Value::as_u64) == Some(3), + LOCK_PATH, + "lockfileVersion is exactly 3", + )?; + let packages = lock.get("packages").and_then(Value::as_object).ok_or( + DocumentationError::RepositoryContract { + path: LOCK_PATH, + requirement: "packages is an object", + }, + )?; + require_package_value( + packages, + "", + &["dependencies", "markdownlint-cli2"], + "packages[\"\"].dependencies.markdownlint-cli2", + "0.23.2", + )?; + require_package_value( + packages, + "node_modules/markdownlint-cli2", + &["dependencies", "js-yaml"], + "packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml", + "5.2.2", + )?; + require_package_value( + packages, + "node_modules/js-yaml", + &["version"], + "packages[\"node_modules/js-yaml\"].version", + "5.2.2", + )?; + require_package_value( + packages, + "node_modules/markdown-it", + &["version"], + "packages[\"node_modules/markdown-it\"].version", + "14.3.0", + )?; + require_provenance(packages)?; + Ok(()) +} + +fn admit_installer(installer: &str) -> Result<(), DocumentationError> { + require( + installer.contains("npm ci"), + INSTALLER_PATH, + "installation uses npm ci", + )?; + require( + installer.contains("package-lock.json"), + INSTALLER_PATH, + "installation requires package-lock.json", + )?; + require( + !installer.contains("npm install \\"), + INSTALLER_PATH, + "installation does not bypass the lock with npm install", + ) +} + +fn parse(path: &'static str, raw: &str) -> Result { + serde_json::from_str(raw).map_err(|source| DocumentationError::RepositoryJson { path, source }) +} + +fn require_package_value( + packages: &Map, + package: &'static str, + fields: &[&str], + field: &'static str, + expected: &'static str, +) -> Result<(), DocumentationError> { + let mut value = packages.get(package); + for field in fields { + value = value.and_then(|current| current.get(field)); + } + let observed = value.and_then(Value::as_str); + if observed == Some(expected) { + Ok(()) + } else { + Err(DocumentationError::RepositoryValue { + path: LOCK_PATH, + field, + expected, + observed: observed.map(str::to_owned), + }) + } +} + +fn require_provenance(packages: &Map) -> Result<(), DocumentationError> { + for (path, package) in packages { + if !path.is_empty() { + let object = package.as_object(); + if !object.is_some_and(|fields| { + fields.contains_key("resolved") && fields.contains_key("integrity") + }) { + return Err(DocumentationError::RepositoryContractAt { + path: LOCK_PATH, + subject: path.clone(), + requirement: "package records resolved and integrity fields", + }); + } + } + } + Ok(()) +} + +fn require( + condition: bool, + path: &'static str, + requirement: &'static str, +) -> Result<(), DocumentationError> { + if condition { + Ok(()) + } else { + Err(DocumentationError::RepositoryContract { path, requirement }) + } +} + +#[cfg(test)] +#[path = "node_toolchain/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs new file mode 100644 index 0000000..9c202bb --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -0,0 +1,119 @@ +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +const MANIFEST: &str = r#"{"dependencies":{"markdownlint-cli2":"0.23.2"}}"#; +const LOCK: &str = r#"{ + "lockfileVersion": 3, + "packages": { + "": {"dependencies": {"markdownlint-cli2": "0.23.2"}}, + "node_modules/markdownlint-cli2": { + "dependencies": {"js-yaml": "5.2.2"}, + "resolved": "example", + "integrity": "example" + }, + "node_modules/js-yaml": { + "version": "5.2.2", + "resolved": "example", + "integrity": "example" + }, + "node_modules/markdown-it": { + "version": "14.3.0", + "resolved": "example", + "integrity": "example" + } + } +}"#; +const INSTALLER: &str = "test -f package-lock.json\nnpm ci\n"; + +#[test] +fn admitted_node_toolchain_is_exact_and_lockfile_installed() { + assert!(super::admit(MANIFEST, LOCK, INSTALLER).is_ok()); +} + +#[test] +fn dependency_overrides_are_refused() { + let manifest = r#"{"overrides":{},"dependencies":{"markdownlint-cli2":"0.23.2"}}"#; + assert!(matches!( + super::admit(manifest, LOCK, INSTALLER), + Err(super::DocumentationError::RepositoryContract { + path: super::MANIFEST_PATH, + requirement: "dependency overrides are absent", + }) + )); +} + +#[test] +fn dependency_version_drift_is_refused() { + let lock = LOCK.replacen("\"5.2.2\"", "\"5.2.1\"", 1); + let error = super::admit(MANIFEST, &lock, INSTALLER); + assert!(matches!( + &error, + Err(super::DocumentationError::RepositoryValue { + path: super::LOCK_PATH, + field: "packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml", + expected: "5.2.2", + observed: Some(observed), + }) if observed == "5.2.1" + )); + assert_eq!( + error.as_ref().map_err(ToString::to_string), + Err(String::from(concat!( + "repository file `scripts/documentation-tools/package-lock.json` requires ", + "`packages[\"node_modules/markdownlint-cli2\"].dependencies.js-yaml` to be ", + "\"5.2.2\"; observed \"5.2.1\"" + ))) + ); +} + +#[test] +fn missing_dependency_coordinate_is_refused_precisely() { + let lock = LOCK.replacen("\"version\": \"14.3.0\"", "\"missing\": \"14.3.0\"", 1); + assert!(matches!( + super::admit(MANIFEST, &lock, INSTALLER), + Err(super::DocumentationError::RepositoryValue { + path: super::LOCK_PATH, + field: "packages[\"node_modules/markdown-it\"].version", + expected: "14.3.0", + observed: None, + }) + )); +} + +#[test] +fn missing_package_provenance_is_refused() { + let lock = LOCK.replacen("\"integrity\": \"example\"", "\"missing\": \"example\"", 1); + let error = super::admit(MANIFEST, &lock, INSTALLER); + assert!(matches!( + &error, + Err(super::DocumentationError::RepositoryContractAt { + path: super::LOCK_PATH, + subject, + requirement: "package records resolved and integrity fields", + }) if subject == "node_modules/markdownlint-cli2" + )); + let diagnostic = error.as_ref().map_err(ToString::to_string); + assert!(diagnostic.is_err_and(|message| message.contains("node_modules/markdownlint-cli2"))); +} + +#[test] +fn unlocked_installer_is_refused() { + assert!(matches!( + super::admit(MANIFEST, LOCK, "npm install markdownlint-cli2\n"), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installation uses npm ci", + }) + )); +} + +#[test] +fn committed_node_toolchain_satisfies_the_rust_law() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + super::check(&repository_root)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} diff --git a/xtask/src/documentation_integrity/repository_text.rs b/xtask/src/documentation_integrity/repository_text.rs new file mode 100644 index 0000000..c82401e --- /dev/null +++ b/xtask/src/documentation_integrity/repository_text.rs @@ -0,0 +1,50 @@ +//! This module owns bounded UTF-8 reads of fixed repository policy files. + +use std::io::Read; +use std::path::Path; + +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; + +use super::error::DocumentationError; + +const MAX_REPOSITORY_FILE_BYTES: u64 = 1_048_576; + +pub(super) fn read( + repository_root: &RepositoryRoot, + path: &'static str, +) -> Result { + let file = repository_root + .open_file(Path::new(path)) + .map_err(|error| open_error(path, error))?; + let read_bound = MAX_REPOSITORY_FILE_BYTES.checked_add(1).ok_or( + DocumentationError::RepositoryFileTooLarge { + path, + maximum: MAX_REPOSITORY_FILE_BYTES, + }, + )?; + let mut bytes = Vec::new(); + file.take(read_bound) + .read_to_end(&mut bytes) + .map_err(|source| DocumentationError::RepositoryFileInspect { path, source })?; + if u64::try_from(bytes.len()).map_or(true, |length| length > MAX_REPOSITORY_FILE_BYTES) { + return Err(DocumentationError::RepositoryFileTooLarge { + path, + maximum: MAX_REPOSITORY_FILE_BYTES, + }); + } + String::from_utf8(bytes) + .map_err(|source| DocumentationError::RepositoryFileEncoding { path, source }) +} + +fn open_error(path: &'static str, error: OpenRepositoryFileError) -> DocumentationError { + match error { + OpenRepositoryFileError::Io(source) => { + DocumentationError::RepositoryFileInspect { path, source } + } + OpenRepositoryFileError::NonRegular => DocumentationError::RepositoryFileNonRegular(path), + } +} + +#[cfg(test)] +#[path = "repository_text/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/repository_text/tests.rs b/xtask/src/documentation_integrity/repository_text/tests.rs new file mode 100644 index 0000000..130a446 --- /dev/null +++ b/xtask/src/documentation_integrity/repository_text/tests.rs @@ -0,0 +1,52 @@ +use std::fs; +use std::io::Write; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::{MAX_REPOSITORY_FILE_BYTES, read}; + +#[test] +fn repository_policy_reads_are_utf8_and_bounded() -> Result<(), Box> { + let directory = TestDirectory::create("repository-text")?; + fs::write(directory.path().join("policy.txt"), "policy\n")?; + let root = RepositoryRoot::open(directory.path())?; + + assert_eq!(read(&root, "policy.txt")?, "policy\n"); + + fs::write(directory.path().join("policy.txt"), [0xff])?; + assert!(matches!( + read(&root, "policy.txt"), + Err(super::DocumentationError::RepositoryFileEncoding { + path: "policy.txt", + .. + }) + )); + drop(root); + directory.close()?; + Ok(()) +} + +#[test] +fn repository_policy_reads_refuse_bytes_beyond_the_bound() -> Result<(), Box> +{ + let directory = TestDirectory::create("repository-text-bound")?; + let path = directory.path().join("policy.txt"); + let mut file = fs::File::create(&path)?; + let maximum = usize::try_from(MAX_REPOSITORY_FILE_BYTES)?; + file.write_all(&vec![b'x'; maximum])?; + file.write_all(b"x")?; + drop(file); + let root = RepositoryRoot::open(directory.path())?; + + assert!(matches!( + read(&root, "policy.txt"), + Err(super::DocumentationError::RepositoryFileTooLarge { + path: "policy.txt", + maximum: MAX_REPOSITORY_FILE_BYTES, + }) + )); + drop(root); + directory.close()?; + Ok(()) +} From 0313369687c2d01e8f585ed56e0d4cab8c2b5065 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:16:57 -0700 Subject: [PATCH 007/113] Remove boolean documentation policy parameter --- .../documentation_integrity/node_toolchain.rs | 61 ++++++++----------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index 3944a1a..8b6c4d1 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -27,19 +27,17 @@ fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), Documentatio } fn admit_manifest(manifest: &Value) -> Result<(), DocumentationError> { - require( - manifest.get("overrides").is_none(), - MANIFEST_PATH, - "dependency overrides are absent", - ) + if manifest.get("overrides").is_none() { + Ok(()) + } else { + Err(contract(MANIFEST_PATH, "dependency overrides are absent")) + } } fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { - require( - lock.get("lockfileVersion").and_then(Value::as_u64) == Some(3), - LOCK_PATH, - "lockfileVersion is exactly 3", - )?; + if lock.get("lockfileVersion").and_then(Value::as_u64) != Some(3) { + return Err(contract(LOCK_PATH, "lockfileVersion is exactly 3")); + } let packages = lock.get("packages").and_then(Value::as_object).ok_or( DocumentationError::RepositoryContract { path: LOCK_PATH, @@ -79,21 +77,22 @@ fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { } fn admit_installer(installer: &str) -> Result<(), DocumentationError> { - require( - installer.contains("npm ci"), - INSTALLER_PATH, - "installation uses npm ci", - )?; - require( - installer.contains("package-lock.json"), - INSTALLER_PATH, - "installation requires package-lock.json", - )?; - require( - !installer.contains("npm install \\"), - INSTALLER_PATH, - "installation does not bypass the lock with npm install", - ) + if !installer.contains("npm ci") { + return Err(contract(INSTALLER_PATH, "installation uses npm ci")); + } + if !installer.contains("package-lock.json") { + return Err(contract( + INSTALLER_PATH, + "installation requires package-lock.json", + )); + } + if installer.contains("npm install \\") { + return Err(contract( + INSTALLER_PATH, + "installation does not bypass the lock with npm install", + )); + } + Ok(()) } fn parse(path: &'static str, raw: &str) -> Result { @@ -142,16 +141,8 @@ fn require_provenance(packages: &Map) -> Result<(), Documentation Ok(()) } -fn require( - condition: bool, - path: &'static str, - requirement: &'static str, -) -> Result<(), DocumentationError> { - if condition { - Ok(()) - } else { - Err(DocumentationError::RepositoryContract { path, requirement }) - } +fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { path, requirement } } #[cfg(test)] From 906223c3e9081af6c12b074c84d3a86ac8d6096b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:18:39 -0700 Subject: [PATCH 008/113] Port Dependabot coverage laws --- xtask/src/documentation_integrity.rs | 1 + .../src/documentation_integrity/dependabot.rs | 176 ++++++++++++++++++ .../dependabot/manifest.rs | 63 +++++++ .../dependabot/tests.rs | 93 +++++++++ 4 files changed, 333 insertions(+) create mode 100644 xtask/src/documentation_integrity/dependabot.rs create mode 100644 xtask/src/documentation_integrity/dependabot/manifest.rs create mode 100644 xtask/src/documentation_integrity/dependabot/tests.rs diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index a33e59c..848b211 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -1,6 +1,7 @@ //! This module owns documentation and workflow integrity orchestration. mod corpus; +mod dependabot; mod error; mod node_toolchain; mod repository_text; diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs new file mode 100644 index 0000000..de04c05 --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -0,0 +1,176 @@ +//! This module owns Dependabot manifest coverage and maintenance policy. + +mod manifest; + +use std::collections::BTreeSet; +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text; +use manifest::tracked_scopes; + +const DEPENDABOT_PATH: &str = ".github/dependabot.yml"; +const UPDATE_MARKER: &str = " - package-ecosystem: "; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct DependencyScope { + ecosystem: String, + directory: String, +} + +pub(super) fn check( + repository_path: &Path, + repository_root: &RepositoryRoot, +) -> Result<(), DocumentationError> { + let raw = repository_text::read(repository_root, DEPENDABOT_PATH)?; + let required = tracked_scopes(repository_path)?; + admit(&raw, &required) +} + +fn admit(raw: &str, required: &BTreeSet) -> Result<(), DocumentationError> { + if !raw.starts_with("version: 2\nupdates:\n") { + return Err(contract("version and updates header is exact")); + } + let blocks = update_blocks(raw); + if blocks.is_empty() { + return Err(contract("at least one update block exists")); + } + let mut configured = BTreeSet::new(); + for block in blocks { + let scopes = block_scopes(&block)?; + admit_maintenance_policy(&block, &scopes)?; + for scope in scopes { + if !configured.insert(scope.clone()) { + return Err(contract_at( + scope.diagnostic(), + "update scope appears exactly once", + )); + } + } + } + if let Some(missing) = required.difference(&configured).next() { + return Err(contract_at( + missing.diagnostic(), + "tracked dependency scope has an update policy", + )); + } + Ok(()) +} + +fn update_blocks(raw: &str) -> Vec> { + let mut blocks = Vec::new(); + let mut block = Vec::new(); + let mut active = false; + for line in raw.lines() { + if line.starts_with(UPDATE_MARKER) { + if active { + blocks.push(std::mem::take(&mut block)); + } + active = true; + } + if active { + block.push(line); + } + } + if active { + blocks.push(block); + } + blocks +} + +fn block_scopes(block: &[&str]) -> Result, DocumentationError> { + let ecosystem = block + .first() + .and_then(|line| line.strip_prefix(UPDATE_MARKER)) + .map(unquote) + .ok_or_else(|| contract("every update block names an ecosystem"))?; + let mut scopes = Vec::new(); + let mut lines = block.iter(); + while let Some(line) = lines.next() { + if let Some(directory) = line.strip_prefix(" directory: ") { + scopes.push(DependencyScope::new(ecosystem, unquote(directory))); + } else if *line == " directories:" { + scopes.extend( + lines + .by_ref() + .map_while(|entry| entry.strip_prefix(" - ")) + .map(|directory| DependencyScope::new(ecosystem, unquote(directory))), + ); + } + } + if scopes.is_empty() { + Err(contract_at( + ecosystem.to_owned(), + "update block names at least one directory", + )) + } else { + Ok(scopes) + } +} + +fn admit_maintenance_policy( + block: &[&str], + scopes: &[DependencyScope], +) -> Result<(), DocumentationError> { + let raw = block.join("\n"); + let uniform = raw.contains(" schedule:\n interval: weekly") + && raw.contains(" open-pull-requests-limit: 5") + && raw.contains(" labels:\n - dependencies"); + if uniform { + Ok(()) + } else { + let subject = scopes + .first() + .map_or_else(String::new, DependencyScope::diagnostic); + Err(contract_at( + subject, + "update block uses the maintenance policy", + )) + } +} + +fn unquote(raw: &str) -> &str { + let bytes = raw.as_bytes(); + match (bytes.first(), bytes.last()) { + (Some(first), Some(last)) + if bytes.len() >= 2 && first == last && matches!(first, b'\'' | b'"') => + { + raw.get(1..raw.len().saturating_sub(1)).unwrap_or(raw) + } + _ => raw, + } +} + +impl DependencyScope { + fn new(ecosystem: &str, directory: &str) -> Self { + Self { + ecosystem: ecosystem.to_owned(), + directory: directory.to_owned(), + } + } + + fn diagnostic(&self) -> String { + format!("{} {}", self.ecosystem, self.directory) + } +} + +fn contract(requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { + path: DEPENDABOT_PATH, + requirement, + } +} + +fn contract_at(subject: String, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContractAt { + path: DEPENDABOT_PATH, + subject, + requirement, + } +} + +#[cfg(test)] +#[path = "dependabot/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/dependabot/manifest.rs b/xtask/src/documentation_integrity/dependabot/manifest.rs new file mode 100644 index 0000000..b559008 --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot/manifest.rs @@ -0,0 +1,63 @@ +//! This module owns tracked dependency-manifest scope discovery. + +use std::collections::BTreeSet; +use std::path::Path; + +use xtask::protocol_admission::posix_relative_path; + +use crate::git_inventory::{GitPath, paths}; + +use super::DependencyScope; +use crate::documentation_integrity::error::DocumentationError; + +const MANIFEST_ARGUMENTS: [&str; 5] = [ + "ls-files", + "-z", + "--", + ":(glob)**/Cargo.toml", + ":(glob)**/package.json", +]; + +pub(super) fn tracked_scopes( + repository_root: &Path, +) -> Result, DocumentationError> { + paths( + repository_root, + &MANIFEST_ARGUMENTS, + "list tracked dependency manifests", + )? + .iter() + .map(manifest_scope) + .chain([Ok(DependencyScope::new("github-actions", "/"))]) + .collect() +} + +fn manifest_scope(path: &GitPath) -> Result { + let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { + DocumentationError::PathEncoding { + corpus: "dependency manifest", + source, + } + })?; + posix_relative_path(&text).map_err(|_| DocumentationError::InvalidPath { + corpus: "dependency manifest", + path: text.clone(), + })?; + let (directory, filename) = text.rsplit_once('/').unwrap_or(("", &text)); + let ecosystem = match filename { + "Cargo.toml" => "cargo", + "package.json" => "npm", + _ => { + return Err(DocumentationError::InvalidPath { + corpus: "dependency manifest", + path: text, + }); + } + }; + let directory = if directory.is_empty() { + String::from("/") + } else { + format!("/{directory}") + }; + Ok(DependencyScope::new(ecosystem, &directory)) +} diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs new file mode 100644 index 0000000..93d10bf --- /dev/null +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -0,0 +1,93 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +use super::DependencyScope; + +const POLICY: &str = r"version: 2 +updates: + - package-ecosystem: cargo + directories: + - / + - /xtask + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: + - dependencies + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + labels: + - dependencies +"; + +fn required() -> BTreeSet { + [ + DependencyScope::new("cargo", "/"), + DependencyScope::new("cargo", "/xtask"), + DependencyScope::new("github-actions", "/"), + ] + .into_iter() + .collect() +} + +#[test] +fn complete_uniform_dependabot_policy_is_admitted() { + assert!(super::admit(POLICY, &required()).is_ok()); +} + +#[test] +fn missing_manifest_scope_is_refused() { + let policy = POLICY.replace(" - /xtask\n", ""); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "tracked dependency scope has an update policy", + }) if subject == "cargo /xtask" + )); +} + +#[test] +fn duplicate_update_scope_is_refused() { + let policy = POLICY.replace(" - /xtask\n", " - /xtask\n - /xtask\n"); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update scope appears exactly once", + }) if subject == "cargo /xtask" + )); +} + +#[test] +fn nonuniform_maintenance_policy_is_refused() { + let policy = POLICY.replacen(" interval: weekly", " interval: daily", 1); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update block uses the maintenance policy", + }) if subject == "cargo /" + )); +} + +#[test] +fn committed_dependabot_policy_covers_every_tracked_manifest() +-> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + super::check(root, &repository_root)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} From aa0c80e89f44ce553a99e24e6fdc8fcb093788ad Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:21:33 -0700 Subject: [PATCH 009/113] Split documentation error rendering --- xtask/src/documentation_integrity/error.rs | 91 +----------------- .../documentation_integrity/error/display.rs | 95 +++++++++++++++++++ 2 files changed, 97 insertions(+), 89 deletions(-) create mode 100644 xtask/src/documentation_integrity/error/display.rs diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 3994349..a34bf49 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -1,11 +1,12 @@ //! This module owns typed documentation-integrity failures. +mod display; + use std::error::Error; use std::fmt; use std::io; use std::string::FromUtf8Error; -use crate::diagnostic::escaped_controls; use crate::git_inventory::GitInventoryError; pub(super) enum DocumentationError { @@ -73,94 +74,6 @@ impl fmt::Debug for DocumentationError { } } -impl fmt::Display for DocumentationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), - Self::GitInventory(error) => write!(formatter, "{error}"), - Self::Inspect { corpus, path, .. } => { - write!(formatter, "cannot inspect {corpus} source `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") - } - Self::InvalidPath { corpus, path } => { - write!(formatter, "{corpus} corpus contains an unsafe path `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") - } - Self::NonRegular { corpus, path } => { - write!(formatter, "{corpus} source is not a regular file: `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") - } - Self::PathEncoding { corpus, .. } => { - write!(formatter, "{corpus} corpus contains a non-UTF-8 path") - } - Self::RepositoryFileEncoding { path, .. } => { - write!(formatter, "repository file `{path}` is not UTF-8") - } - Self::RepositoryFileInspect { path, .. } => { - write!(formatter, "cannot inspect repository file `{path}`") - } - Self::RepositoryFileNonRegular(path) => { - write!(formatter, "repository file is not regular: `{path}`") - } - Self::RepositoryFileTooLarge { path, maximum } => write!( - formatter, - "repository file `{path}` exceeds the {maximum}-byte bound" - ), - Self::RepositoryContract { path, requirement } => { - write!( - formatter, - "repository file `{path}` violates: {requirement}" - ) - } - Self::RepositoryContractAt { - path, - subject, - requirement, - } => { - write!( - formatter, - "repository file `{path}` violates {requirement} at `" - )?; - escaped_controls(formatter, subject)?; - formatter.write_str("`") - } - Self::RepositoryJson { path, .. } => { - write!(formatter, "repository file `{path}` is not valid JSON") - } - Self::RepositoryValue { - path, - field, - expected, - observed, - } => { - write!( - formatter, - "repository file `{path}` requires `{field}` to be {expected:?}; observed " - )?; - match observed { - Some(value) => { - formatter.write_str("\"")?; - escaped_controls(formatter, value)?; - formatter.write_str("\"") - } - None => formatter.write_str("missing"), - } - } - Self::VersionMismatch { - program, - expected, - observed, - } => write!( - formatter, - "{program} version mismatch: expected {expected:?}, observed {observed:?}" - ), - } - } -} - impl Error for DocumentationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs new file mode 100644 index 0000000..37f5d18 --- /dev/null +++ b/xtask/src/documentation_integrity/error/display.rs @@ -0,0 +1,95 @@ +//! This module owns human-readable documentation-integrity diagnostics. + +use std::fmt; + +use crate::diagnostic::escaped_controls; + +use super::DocumentationError; + +impl fmt::Display for DocumentationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), + Self::GitInventory(error) => write!(formatter, "{error}"), + Self::Inspect { corpus, path, .. } => { + write!(formatter, "cannot inspect {corpus} source `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::InvalidPath { corpus, path } => { + write!(formatter, "{corpus} corpus contains an unsafe path `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::NonRegular { corpus, path } => { + write!(formatter, "{corpus} source is not a regular file: `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } + Self::PathEncoding { corpus, .. } => { + write!(formatter, "{corpus} corpus contains a non-UTF-8 path") + } + Self::RepositoryFileEncoding { path, .. } => { + write!(formatter, "repository file `{path}` is not UTF-8") + } + Self::RepositoryFileInspect { path, .. } => { + write!(formatter, "cannot inspect repository file `{path}`") + } + Self::RepositoryFileNonRegular(path) => { + write!(formatter, "repository file is not regular: `{path}`") + } + Self::RepositoryFileTooLarge { path, maximum } => write!( + formatter, + "repository file `{path}` exceeds the {maximum}-byte bound" + ), + Self::RepositoryContract { path, requirement } => { + write!( + formatter, + "repository file `{path}` violates: {requirement}" + ) + } + Self::RepositoryContractAt { + path, + subject, + requirement, + } => { + write!( + formatter, + "repository file `{path}` violates {requirement} at `" + )?; + escaped_controls(formatter, subject)?; + formatter.write_str("`") + } + Self::RepositoryJson { path, .. } => { + write!(formatter, "repository file `{path}` is not valid JSON") + } + Self::RepositoryValue { + path, + field, + expected, + observed, + } => { + write!( + formatter, + "repository file `{path}` requires `{field}` to be {expected:?}; observed " + )?; + match observed { + Some(value) => { + formatter.write_str("\"")?; + escaped_controls(formatter, value)?; + formatter.write_str("\"") + } + None => formatter.write_str("missing"), + } + } + Self::VersionMismatch { + program, + expected, + observed, + } => write!( + formatter, + "{program} version mismatch: expected {expected:?}, observed {observed:?}" + ), + } + } +} From e70b4c079327107731dd79172c58658216b74b86 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:28:14 -0700 Subject: [PATCH 010/113] Ignore repository-local Graft state --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 21b47e8..9c1c290 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ /scripts/documentation-tools/node_modules/ **/*.rs.bk +# Agent workspace state +.graft/ + # Python checker bytecode __pycache__/ *.py[cod] From 4911f698e2df77b2730c1468c0e0f069aec0637f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:28:55 -0700 Subject: [PATCH 011/113] Add bounded documentation integrity command --- xtask/src/bounded_process.rs | 3 + xtask/src/bounded_process/error.rs | 13 ++ xtask/src/documentation_integrity.rs | 39 ++++ .../src/documentation_integrity/dependabot.rs | 4 +- xtask/src/documentation_integrity/error.rs | 32 +++- .../documentation_integrity/error/display.rs | 167 ++++++++++++++---- .../src/documentation_integrity/execution.rs | 155 ++++++++++++++++ .../execution/tests.rs | 150 ++++++++++++++++ .../documentation_integrity/node_toolchain.rs | 2 +- xtask/src/fuzz_campaign/execution/tests.rs | 1 + xtask/src/main.rs | 8 +- xtask/src/task_error.rs | 13 +- xtask/tests/cli_contract.rs | 3 +- 13 files changed, 551 insertions(+), 39 deletions(-) create mode 100644 xtask/src/documentation_integrity/execution.rs create mode 100644 xtask/src/documentation_integrity/execution/tests.rs diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 4c439ec..21dfb53 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -14,6 +14,7 @@ use crate::process_output::{BoundedBytes, bounded_bytes}; const OUTPUT_LIMIT: usize = 1_048_576; pub(crate) struct ProcessOutput { + pub(crate) code: Option, pub(crate) succeeded: bool, pub(crate) stdout: Vec, pub(crate) stderr: Vec, @@ -29,6 +30,7 @@ pub(crate) fn status( source, })?; Ok(ProcessOutput { + code: status.code(), succeeded: status.success(), stdout: Vec::new(), stderr: Vec::new(), @@ -79,6 +81,7 @@ pub(crate) fn capture( refuse_exceeded(program, "stdout", &stdout)?; refuse_exceeded(program, "stderr", &stderr)?; Ok(ProcessOutput { + code: status.code(), succeeded: status.success(), stdout: stdout.bytes, stderr: stderr.bytes, diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs index 3f89730..d4e16eb 100644 --- a/xtask/src/bounded_process/error.rs +++ b/xtask/src/bounded_process/error.rs @@ -35,6 +35,19 @@ pub(crate) enum ProcessError { }, } +impl ProcessError { + pub(crate) fn is_not_found(&self) -> bool { + match self { + Self::Cleanup { primary, .. } => primary.is_not_found(), + Self::Io { source, .. } => source.kind() == io::ErrorKind::NotFound, + Self::MissingStream { .. } + | Self::OutputLimit { .. } + | Self::ReaderPanic { .. } + | Self::Timeout { .. } => false, + } + } +} + impl fmt::Debug for ProcessError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self, formatter) diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 848b211..654d33f 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -3,6 +3,45 @@ mod corpus; mod dependabot; mod error; +mod execution; mod node_toolchain; mod repository_text; mod tool; + +use std::path::{Path, PathBuf}; + +use crate::repository_file::RepositoryRoot; + +pub(super) use error::DocumentationError; + +pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { + let repository_root = RepositoryRoot::open(repository_path).map_err(|source| { + DocumentationError::RepositoryRootInspect { + path: repository_path.to_owned(), + source, + } + })?; + verify_root(&repository_root, repository_path)?; + node_toolchain::check(&repository_root)?; + dependabot::check(repository_path, &repository_root)?; + let markdown = corpus::SourceCorpus::markdown(repository_path)?; + let workflows = corpus::SourceCorpus::workflow(repository_path)?; + execution::run(repository_path, markdown.paths(), workflows.paths())?; + verify_root(&repository_root, repository_path) +} + +fn verify_root( + repository_root: &RepositoryRoot, + repository_path: &Path, +) -> Result<(), DocumentationError> { + match repository_root.is_current_path() { + Ok(true) => Ok(()), + Ok(false) => Err(DocumentationError::RepositoryRootChanged( + repository_path.to_owned(), + )), + Err(source) => Err(DocumentationError::RepositoryRootInspect { + path: PathBuf::from(repository_path), + source, + }), + } +} diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index de04c05..198b945 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -156,14 +156,14 @@ impl DependencyScope { } } -fn contract(requirement: &'static str) -> DocumentationError { +const fn contract(requirement: &'static str) -> DocumentationError { DocumentationError::RepositoryContract { path: DEPENDABOT_PATH, requirement, } } -fn contract_at(subject: String, requirement: &'static str) -> DocumentationError { +const fn contract_at(subject: String, requirement: &'static str) -> DocumentationError { DocumentationError::RepositoryContractAt { path: DEPENDABOT_PATH, subject, diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index a34bf49..8db0f92 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -5,11 +5,13 @@ mod display; use std::error::Error; use std::fmt; use std::io; +use std::path::PathBuf; use std::string::FromUtf8Error; +use crate::bounded_process::ProcessError; use crate::git_inventory::GitInventoryError; -pub(super) enum DocumentationError { +pub(crate) enum DocumentationError { EmptyCorpus(&'static str), GitInventory(GitInventoryError), Inspect { @@ -29,6 +31,7 @@ pub(super) enum DocumentationError { corpus: &'static str, source: FromUtf8Error, }, + Process(ProcessError), RepositoryFileEncoding { path: &'static str, source: FromUtf8Error, @@ -55,6 +58,11 @@ pub(super) enum DocumentationError { path: &'static str, source: serde_json::Error, }, + RepositoryRootChanged(PathBuf), + RepositoryRootInspect { + path: PathBuf, + source: io::Error, + }, RepositoryValue { path: &'static str, field: &'static str, @@ -66,6 +74,22 @@ pub(super) enum DocumentationError { expected: &'static str, observed: String, }, + ToolFailed { + program: &'static str, + code: Option, + stdout: String, + stderr: String, + }, + ToolOutputEncoding { + program: &'static str, + stream: &'static str, + source: FromUtf8Error, + }, + ToolUnavailable { + program: &'static str, + install_version: &'static str, + source: ProcessError, + }, } impl fmt::Debug for DocumentationError { @@ -78,6 +102,7 @@ impl Error for DocumentationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::GitInventory(error) => Some(error), + Self::Process(error) => Some(error), Self::Inspect { source, .. } | Self::RepositoryFileInspect { source, .. } => { Some(source) } @@ -85,6 +110,9 @@ impl Error for DocumentationError { Some(source) } Self::RepositoryJson { source, .. } => Some(source), + Self::RepositoryRootInspect { source, .. } => Some(source), + Self::ToolOutputEncoding { source, .. } => Some(source), + Self::ToolUnavailable { source, .. } => Some(source), Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } @@ -92,7 +120,9 @@ impl Error for DocumentationError { | Self::RepositoryFileTooLarge { .. } | Self::RepositoryContract { .. } | Self::RepositoryContractAt { .. } + | Self::RepositoryRootChanged(_) | Self::RepositoryValue { .. } + | Self::ToolFailed { .. } | Self::VersionMismatch { .. } => None, } } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 37f5d18..07bbcaf 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -2,33 +2,41 @@ use std::fmt; -use crate::diagnostic::escaped_controls; +use crate::diagnostic::{escaped_controls, escaped_path}; use super::DocumentationError; +#[derive(Clone, Copy)] +enum SourcePathDiagnostic { + Inspect, + Invalid, + NonRegular, +} + +#[derive(Clone, Copy)] +enum RepositoryRootDiagnostic { + Changed, + Inspect, +} + impl fmt::Display for DocumentationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), Self::GitInventory(error) => write!(formatter, "{error}"), Self::Inspect { corpus, path, .. } => { - write!(formatter, "cannot inspect {corpus} source `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") + source_path(formatter, SourcePathDiagnostic::Inspect, corpus, path) } Self::InvalidPath { corpus, path } => { - write!(formatter, "{corpus} corpus contains an unsafe path `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") + source_path(formatter, SourcePathDiagnostic::Invalid, corpus, path) } Self::NonRegular { corpus, path } => { - write!(formatter, "{corpus} source is not a regular file: `")?; - escaped_controls(formatter, path)?; - formatter.write_str("`") + source_path(formatter, SourcePathDiagnostic::NonRegular, corpus, path) } Self::PathEncoding { corpus, .. } => { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } + Self::Process(error) => write!(formatter, "{error}"), Self::RepositoryFileEncoding { path, .. } => { write!(formatter, "repository file `{path}` is not UTF-8") } @@ -52,36 +60,22 @@ impl fmt::Display for DocumentationError { path, subject, requirement, - } => { - write!( - formatter, - "repository file `{path}` violates {requirement} at `" - )?; - escaped_controls(formatter, subject)?; - formatter.write_str("`") - } + } => repository_contract_at(formatter, path, subject, requirement), Self::RepositoryJson { path, .. } => { write!(formatter, "repository file `{path}` is not valid JSON") } + Self::RepositoryRootChanged(path) => { + repository_root(formatter, RepositoryRootDiagnostic::Changed, path) + } + Self::RepositoryRootInspect { path, .. } => { + repository_root(formatter, RepositoryRootDiagnostic::Inspect, path) + } Self::RepositoryValue { path, field, expected, observed, - } => { - write!( - formatter, - "repository file `{path}` requires `{field}` to be {expected:?}; observed " - )?; - match observed { - Some(value) => { - formatter.write_str("\"")?; - escaped_controls(formatter, value)?; - formatter.write_str("\"") - } - None => formatter.write_str("missing"), - } - } + } => repository_value(formatter, path, field, expected, observed.as_deref()), Self::VersionMismatch { program, expected, @@ -90,6 +84,115 @@ impl fmt::Display for DocumentationError { formatter, "{program} version mismatch: expected {expected:?}, observed {observed:?}" ), + Self::ToolFailed { + program, + code, + stdout, + stderr, + } => tool_failed(formatter, program, *code, stdout, stderr), + Self::ToolOutputEncoding { + program, stream, .. + } => { + write!(formatter, "{program} {stream} is not UTF-8") + } + Self::ToolUnavailable { + program, + install_version, + .. + } => write!( + formatter, + "{program} is unavailable; install version {install_version}" + ), + } + } +} + +fn source_path( + formatter: &mut fmt::Formatter<'_>, + diagnostic: SourcePathDiagnostic, + corpus: &str, + path: &str, +) -> fmt::Result { + match diagnostic { + SourcePathDiagnostic::Inspect => write!(formatter, "cannot inspect {corpus} source `")?, + SourcePathDiagnostic::Invalid => { + write!(formatter, "{corpus} corpus contains an unsafe path `")?; + } + SourcePathDiagnostic::NonRegular => { + write!(formatter, "{corpus} source is not a regular file: `")?; + } + } + escaped_controls(formatter, path)?; + formatter.write_str("`") +} + +fn repository_contract_at( + formatter: &mut fmt::Formatter<'_>, + path: &str, + subject: &str, + requirement: &str, +) -> fmt::Result { + write!( + formatter, + "repository file `{path}` violates {requirement} at `" + )?; + escaped_controls(formatter, subject)?; + formatter.write_str("`") +} + +fn repository_root( + formatter: &mut fmt::Formatter<'_>, + diagnostic: RepositoryRootDiagnostic, + path: &std::path::Path, +) -> fmt::Result { + match diagnostic { + RepositoryRootDiagnostic::Changed => { + formatter.write_str("documentation repository root changed: `")?; + } + RepositoryRootDiagnostic::Inspect => { + formatter.write_str("cannot inspect documentation repository root: `")?; + } + } + escaped_path(formatter, path)?; + formatter.write_str("`") +} + +fn repository_value( + formatter: &mut fmt::Formatter<'_>, + path: &str, + field: &str, + expected: &str, + observed: Option<&str>, +) -> fmt::Result { + write!( + formatter, + "repository file `{path}` requires `{field}` to be {expected:?}; observed " + )?; + match observed { + Some(value) => { + formatter.write_str("\"")?; + escaped_controls(formatter, value)?; + formatter.write_str("\"") } + None => formatter.write_str("missing"), + } +} + +fn tool_failed( + formatter: &mut fmt::Formatter<'_>, + program: &str, + code: Option, + stdout: &str, + stderr: &str, +) -> fmt::Result { + write!(formatter, "{program} failed with exit status ")?; + match code { + Some(code) => write!(formatter, "{code}")?, + None => formatter.write_str("unavailable")?, } + formatter.write_str("; stdout \"")?; + escaped_controls(formatter, stdout)?; + formatter.write_str("\"; stderr \"")?; + escaped_controls(formatter, stderr)?; + formatter.write_str("\"") } diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs new file mode 100644 index 0000000..7dd94da --- /dev/null +++ b/xtask/src/documentation_integrity/execution.rs @@ -0,0 +1,155 @@ +//! This module owns bounded execution of admitted documentation tools. + +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::bounded_process::{self, ProcessOutput}; + +use super::error::DocumentationError; +use super::tool::DocumentationTool; + +const TOOL_DEADLINE: Duration = Duration::from_mins(2); + +trait ToolRunner { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result; +} + +struct ExternalToolRunner<'a> { + repository_root: &'a Path, +} + +pub(super) fn run( + repository_root: &Path, + markdown: &[String], + workflows: &[String], +) -> Result<(), DocumentationError> { + run_with( + &mut ExternalToolRunner { repository_root }, + markdown, + workflows, + ) +} + +fn run_with( + runner: &mut impl ToolRunner, + markdown: &[String], + workflows: &[String], +) -> Result<(), DocumentationError> { + admit_version(runner, DocumentationTool::Markdownlint)?; + admit_version(runner, DocumentationTool::Lychee)?; + let lint = run_check(runner, DocumentationTool::Markdownlint, markdown); + let links = run_check(runner, DocumentationTool::Lychee, markdown); + lint?; + links?; + admit_version(runner, DocumentationTool::Actionlint)?; + run_check(runner, DocumentationTool::Actionlint, workflows) +} + +fn admit_version( + runner: &mut impl ToolRunner, + tool: DocumentationTool, +) -> Result<(), DocumentationError> { + let arguments = text_arguments(tool.version_arguments()); + let output = runner.capture(tool, &arguments)?; + let stdout = String::from_utf8(output.stdout).map_err(|source| { + DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "version stdout", + source, + } + })?; + let observed = stdout.lines().next().unwrap_or(""); + let admission = tool.admit_version(observed); + if output.succeeded { + admission + } else { + admission?; + Err(failed(tool, output.code, stdout, output.stderr)?) + } +} + +fn run_check( + runner: &mut impl ToolRunner, + tool: DocumentationTool, + paths: &[String], +) -> Result<(), DocumentationError> { + let mut arguments = text_arguments(tool.check_prefix()); + arguments.extend(paths.iter().cloned()); + let output = runner.capture(tool, &arguments)?; + if output.succeeded { + Ok(()) + } else { + Err(failed( + tool, + output.code, + String::from_utf8(output.stdout).map_err(|source| { + DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "stdout", + source, + } + })?, + output.stderr, + )?) + } +} + +fn failed( + tool: DocumentationTool, + code: Option, + stdout: String, + stderr: Vec, +) -> Result { + let stderr = + String::from_utf8(stderr).map_err(|source| DocumentationError::ToolOutputEncoding { + program: tool.program(), + stream: "stderr", + source, + })?; + Ok(DocumentationError::ToolFailed { + program: tool.program(), + code, + stdout, + stderr, + }) +} + +fn text_arguments(arguments: &[&str]) -> Vec { + arguments.iter().map(ToString::to_string).collect() +} + +impl ToolRunner for ExternalToolRunner<'_> { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + let mut command = Command::new(tool.program()); + command + .args(arguments) + .current_dir(self.repository_root) + .stdin(Stdio::null()); + bounded_process::capture(tool.program(), &mut command, Some(TOOL_DEADLINE)).map_err( + |source| { + if source.is_not_found() { + DocumentationError::ToolUnavailable { + program: tool.program(), + install_version: tool.install_version(), + source, + } + } else { + DocumentationError::Process(source) + } + }, + ) + } +} + +#[cfg(test)] +#[path = "execution/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs new file mode 100644 index 0000000..5e421ca --- /dev/null +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -0,0 +1,150 @@ +use std::collections::VecDeque; + +use crate::bounded_process::ProcessOutput; + +use super::{DocumentationError, DocumentationTool, ToolRunner}; + +struct RecordingRunner { + calls: Vec<(DocumentationTool, Vec)>, + outputs: VecDeque, +} + +#[test] +fn admitted_tools_run_with_exact_arguments_and_silent_success() { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + success(), + success(), + version(DocumentationTool::Actionlint), + success(), + ]); + let markdown = [String::from("README.md")]; + let workflows = [String::from(".github/workflows/ci.yml")]; + + assert!(super::run_with(&mut runner, &markdown, &workflows).is_ok()); + assert_eq!(runner.calls.len(), 6); + assert_eq!( + runner.calls.get(2), + Some(&( + DocumentationTool::Markdownlint, + vec![ + String::from("--no-globs"), + String::from("--"), + String::from("README.md") + ] + )) + ); + assert_eq!( + runner.calls.get(5), + Some(&( + DocumentationTool::Actionlint, + vec![ + String::from("-shellcheck="), + String::from("-pyflakes="), + String::from(".github/workflows/ci.yml") + ] + )) + ); +} + +#[test] +fn link_check_runs_after_markdownlint_returns_nonzero() { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + failure(b"lint", b""), + success(), + ]); + + let result = super::run_with(&mut runner, &[String::from("README.md")], &[]); + + assert!(matches!( + result, + Err(DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + ref stdout, + ref stderr, + }) if stdout == "lint" && stderr.is_empty() + )); + assert_eq!(runner.calls.len(), 4); + assert_eq!( + runner.calls.get(3).map(|call| call.0), + Some(DocumentationTool::Lychee) + ); +} + +#[test] +fn unreviewed_version_stops_before_tool_execution() { + let mut runner = RecordingRunner::new([ProcessOutput { + code: Some(0), + succeeded: true, + stdout: b"markdownlint-cli2 v999.0.0\n".to_vec(), + stderr: Vec::new(), + }]); + + let result = super::run_with(&mut runner, &[], &[]); + + assert!(matches!( + result, + Err(DocumentationError::VersionMismatch { + program: "markdownlint-cli2", + expected: "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + ref observed, + }) if observed == "markdownlint-cli2 v999.0.0" + )); + assert_eq!(runner.calls.len(), 1); +} + +impl RecordingRunner { + fn new(outputs: impl IntoIterator) -> Self { + Self { + calls: Vec::new(), + outputs: outputs.into_iter().collect(), + } + } +} + +impl ToolRunner for RecordingRunner { + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + self.calls.push((tool, arguments.to_vec())); + self.outputs + .pop_front() + .ok_or(DocumentationError::RepositoryContract { + path: "test runner", + requirement: "one output exists for every expected call", + }) + } +} + +fn version(tool: DocumentationTool) -> ProcessOutput { + ProcessOutput { + code: Some(0), + succeeded: true, + stdout: format!("{}\n", tool.expected_version()).into_bytes(), + stderr: Vec::new(), + } +} + +const fn success() -> ProcessOutput { + ProcessOutput { + code: Some(0), + succeeded: true, + stdout: Vec::new(), + stderr: Vec::new(), + } +} + +fn failure(stdout: &[u8], stderr: &[u8]) -> ProcessOutput { + ProcessOutput { + code: Some(1), + succeeded: false, + stdout: stdout.to_vec(), + stderr: stderr.to_vec(), + } +} diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index 8b6c4d1..842c131 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -141,7 +141,7 @@ fn require_provenance(packages: &Map) -> Result<(), Documentation Ok(()) } -fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { +const fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { DocumentationError::RepositoryContract { path, requirement } } diff --git a/xtask/src/fuzz_campaign/execution/tests.rs b/xtask/src/fuzz_campaign/execution/tests.rs index 0186a18..a25fec5 100644 --- a/xtask/src/fuzz_campaign/execution/tests.rs +++ b/xtask/src/fuzz_campaign/execution/tests.rs @@ -83,6 +83,7 @@ fn policy() -> Result> { fn output(succeeded: bool, stdout: &[u8], stderr: &[u8]) -> ProcessOutput { ProcessOutput { + code: Some(i32::from(!succeeded)), succeeded, stdout: stdout.to_vec(), stderr: stderr.to_vec(), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e16766b..70deeda 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -14,7 +14,10 @@ mod benchmark_baseline; )] mod bounded_process; mod diagnostic; -#[cfg(test)] +#[allow( + clippy::redundant_pub_crate, + reason = "the command and task-error boundaries are sibling consumers" +)] mod documentation_integrity; #[allow( clippy::redundant_pub_crate, @@ -106,6 +109,9 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { "conformance-check" => { protocol_conformance::check(repository_root)?; } + "documentation-integrity-check" => { + documentation_integrity::check(repository_root)?; + } "prepare-fuzz-corpus" => { fuzz_seed_corpus::prepare(repository_root)?; } diff --git a/xtask/src/task_error.rs b/xtask/src/task_error.rs index e839669..297a4f5 100644 --- a/xtask/src/task_error.rs +++ b/xtask/src/task_error.rs @@ -5,6 +5,7 @@ use std::fmt; use crate::benchmark_baseline::BenchmarkBaselineError; use crate::diagnostic::escaped_controls; +use crate::documentation_integrity::DocumentationError; use crate::fuzz_campaign::FuzzCampaignError; use crate::fuzz_seed_corpus::FuzzSeedError; use crate::golden_file_worldline::GoldenError; @@ -14,6 +15,7 @@ use crate::source_structure::SourceStructureError; pub(super) enum TaskError { BenchmarkBaseline(BenchmarkBaselineError), Conformance(ConformanceError), + Documentation(DocumentationError), FuzzCampaign(FuzzCampaignError), FuzzSeed(FuzzSeedError), Golden(GoldenError), @@ -37,6 +39,7 @@ impl fmt::Display for TaskError { match self { Self::BenchmarkBaseline(error) => write!(formatter, "{error}"), Self::Conformance(error) => write!(formatter, "{error}"), + Self::Documentation(error) => write!(formatter, "{error}"), Self::FuzzCampaign(error) => write!(formatter, "{error}"), Self::FuzzSeed(error) => write!(formatter, "{error}"), Self::Golden(error) => write!(formatter, "{error}"), @@ -62,7 +65,8 @@ impl fmt::Display for TaskError { "usage: cargo xtask \ ", ), } @@ -74,6 +78,7 @@ impl Error for TaskError { match self { Self::BenchmarkBaseline(error) => Some(error), Self::Conformance(error) => Some(error), + Self::Documentation(error) => Some(error), Self::FuzzCampaign(error) => Some(error), Self::FuzzSeed(error) => Some(error), Self::Golden(error) => Some(error), @@ -100,6 +105,12 @@ impl From for TaskError { } } +impl From for TaskError { + fn from(error: DocumentationError) -> Self { + Self::Documentation(error) + } +} + impl From for TaskError { fn from(error: FuzzCampaignError) -> Self { Self::FuzzCampaign(error) diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index b52bc3a..74802ae 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -96,7 +96,8 @@ fn missing_command_returns_the_versioned_usage_contract() -> Result<(), io::Erro b"Error: usage: cargo xtask \ \n" ); Ok(()) From d67c970bb9db4f90327980e5495ac2589ca6110d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:39:37 -0700 Subject: [PATCH 012/113] Replace documentation Python with xtask --- .github/workflows/ci.yml | 16 +- .gitignore | 4 - CHANGELOG.md | 4 + CONTRIBUTING.md | 16 +- docs/Documentation Standards.md | 36 ++-- scripts/check_markdown.py | 147 --------------- scripts/check_workflows.py | 104 ----------- scripts/test_check_markdown.py | 132 -------------- scripts/test_dependabot_coverage.py | 120 ------------- scripts/test_documentation_integrity.py | 167 ------------------ scripts/test_documentation_refusals.py | 109 ------------ scripts/test_workflow_corpus.py | 147 --------------- xtask/src/documentation_integrity.rs | 2 + .../src/documentation_integrity/execution.rs | 3 + .../execution/external_tests.rs | 79 +++++++++ .../workflow_contract.rs | 61 +++++++ .../workflow_contract/tests.rs | 47 +++++ xtask/src/main.rs | 1 + xtask/src/source_structure.rs | 30 ++-- xtask/src/source_structure/pure_rust_tests.rs | 14 ++ xtask/src/source_structure/source_error.rs | 7 + xtask/src/source_structure/source_kind.rs | 24 +++ xtask/tests/cli_contract.rs | 9 - .../tests/cli_contract/documentation_tools.rs | 126 +++++++++++++ xtask/tests/documentation_cli_contract.rs | 20 +++ 25 files changed, 435 insertions(+), 990 deletions(-) delete mode 100644 scripts/check_markdown.py delete mode 100644 scripts/check_workflows.py delete mode 100644 scripts/test_check_markdown.py delete mode 100644 scripts/test_dependabot_coverage.py delete mode 100644 scripts/test_documentation_integrity.py delete mode 100644 scripts/test_documentation_refusals.py delete mode 100644 scripts/test_workflow_corpus.py create mode 100644 xtask/src/documentation_integrity/execution/external_tests.rs create mode 100644 xtask/src/documentation_integrity/workflow_contract.rs create mode 100644 xtask/src/documentation_integrity/workflow_contract/tests.rs create mode 100644 xtask/src/source_structure/pure_rust_tests.rs create mode 100644 xtask/src/source_structure/source_kind.rs create mode 100644 xtask/tests/cli_contract/documentation_tools.rs create mode 100644 xtask/tests/documentation_cli_contract.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e1fd53..6c3c3ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,6 +82,9 @@ jobs: with: persist-credentials: false + - name: Install pinned toolchain + run: rustup show + - name: Install pinned Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -95,14 +98,13 @@ jobs: "$documentation_tools/bin" \ "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" - - name: Verify documentation integrity laws - run: python3 -m unittest discover -s scripts -p 'test_*.py' -v - - - name: Check Markdown and internal links - run: python3 scripts/check_markdown.py + - name: Verify malformed-input refusal laws + run: | + cargo test --locked --package xtask \ + documentation_integrity::execution::external_tests -- --ignored - - name: Check workflow syntax - run: python3 scripts/check_workflows.py + - name: Check documentation and workflows + run: cargo xtask documentation-integrity-check - name: Check repository whitespace run: git diff --check "$(git hash-object -t tree /dev/null)" HEAD diff --git a/.gitignore b/.gitignore index 9c1c290..f34925c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,6 @@ # Agent workspace state .graft/ -# Python checker bytecode -__pycache__/ -*.py[cod] - # macOS .DS_Store .AppleDouble diff --git a/CHANGELOG.md b/CHANGELOG.md index 192c48f..bb09e36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ after its public API and format compatibility policies are established. ### Changed +- Documentation corpus selection, pinned tool admission, Markdown and fragment + checks, workflow linting, Dependabot coverage, and Node lock-graph policy now + run through bounded Rust `xtask` code; CI and `cargo xtask verify` use that + boundary, and the seven superseded Python checkers have been removed. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bfea61..2e6db04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,18 +38,18 @@ npm ci --prefix scripts/documentation-tools --ignore-scripts --no-audit --no-fun cargo install lychee --version 0.21.0 --locked go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 export PATH="$PWD/scripts/documentation-tools/node_modules/.bin:$PATH" -python3 scripts/check_markdown.py -python3 scripts/check_workflows.py +cargo xtask documentation-integrity-check git diff --check git diff --cached --check ``` -The checker admits tracked Markdown plus nonignored new Markdown and refuses -any other tool version. Build products, generated Rustdoc, fuzz artifacts, -and other ignored files therefore cannot change the result. Link validation -checks local files and fragments with network access disabled; external-site -availability cannot change the result. The two Git commands check unstaged -and staged whitespace errors separately. +The Rust checker admits tracked Markdown and workflows plus nonignored new +files, verifies the committed Node lock graph and Dependabot coverage, and +refuses any other tool version. Build products, generated Rustdoc, fuzz +artifacts, and other ignored files therefore cannot change the result. Link +validation checks local files and fragments with network access disabled; +external-site availability cannot change the result. The two Git commands +check unstaged and staged whitespace errors separately. ## Development checks diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index cb1a37a..ac32c8a 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -533,35 +533,31 @@ Documentation quality requires both deterministic checks and human judgment. Run for documentation changes: ```bash -python3 scripts/check_markdown.py +cargo xtask documentation-integrity-check git diff --check git diff --cached --check ``` -Use `markdownlint-cli2` 0.23.2. The repository-owned configuration defines -the default Markdown input set, imports `.gitignore`, and records deliberate -rule choices. The checker admits tracked Markdown plus nonignored new -Markdown, disables config globs for that invocation, and refuses a different -tool version. It also runs `lychee` 0.21.0 offline with fragment checking, so -external-site availability cannot affect the result. Run it from the -repository root. The two Git commands inspect unstaged and staged whitespace -errors separately. - -When workflows change, also run: - -```bash -python3 scripts/check_workflows.py -``` - -The workflow checker requires `actionlint` 1.7.12 and refuses another version. +Use `markdownlint-cli2` 0.23.2. The repository-owned configuration records +deliberate rule choices. The Rust checker selects tracked Markdown plus +nonignored new Markdown, disables configuration globs for that invocation, +and refuses a different tool version. It also runs `lychee` 0.21.0 offline +with fragment checking, so external-site availability cannot affect the +result. Run it from the repository root. The two Git commands inspect +unstaged and staged whitespace errors separately. + +The same Rust command checks workflows with `actionlint` 1.7.12 and refuses +another version. It also verifies the committed Node lock graph, Dependabot +manifest coverage, and the documentation job's delegation to this command. The dedicated `documentation` job in `.github/workflows/ci.yml` installs the -pinned tools, runs these repository-owned checks, and verifies repository -whitespace before admitting the result as CI evidence. +pinned tools, runs malformed-input refusal laws and the repository-owned +command, and verifies repository whitespace before admitting the result as CI +evidence. CI SHOULD block on facts it can determine reliably: - malformed Markdown; -- broken internal links and anchors, once link checking is available; +- broken internal links and anchors; - failed doctests declared runnable (`cargo test --workspace --doc --locked`); - stale generated reference — public API documentation that no longer diff --git a/scripts/check_markdown.py b/scripts/check_markdown.py deleted file mode 100644 index f6cfe9d..0000000 --- a/scripts/check_markdown.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 -"""Check the Git-admitted Markdown corpus and its internal links.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -import sys - -EXPECTED_LINTER_VERSION = "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)" -EXPECTED_LINK_CHECKER_VERSION = "lychee 0.21.0" - - -def find_linter() -> str: - """Return the Markdown linter path or report a precise setup failure.""" - executable = shutil.which("markdownlint-cli2") - if executable is None: - raise RuntimeError( - "markdownlint-cli2 is unavailable; install version 0.23.2" - ) - return executable - - -def find_link_checker() -> str: - """Return the link checker path or report a precise setup failure.""" - executable = shutil.which("lychee") - if executable is None: - raise RuntimeError("lychee is unavailable; install version 0.21.0") - return executable - - -def verify_linter_version(executable: str) -> None: - """Refuse a Markdown linter version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "--no-globs", "--version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_LINTER_VERSION: - raise RuntimeError( - f"markdownlint-cli2 version mismatch: " - f"expected {EXPECTED_LINTER_VERSION!r}, observed {observed!r}" - ) - - -def verify_link_checker_version(executable: str) -> None: - """Refuse a link checker version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "--version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_LINK_CHECKER_VERSION: - raise RuntimeError( - f"lychee version mismatch: " - f"expected {EXPECTED_LINK_CHECKER_VERSION!r}, " - f"observed {observed!r}" - ) - - -def admit_source_path(path: str) -> bool: - """Admit one existing regular file without following links.""" - try: - mode = os.lstat(path).st_mode - except FileNotFoundError: - return False - except OSError as error: - raise RuntimeError( - f"cannot inspect Markdown source {path!r}: {error}" - ) from error - if not stat.S_ISREG(mode): - raise RuntimeError(f"Markdown source is not a regular file: {path!r}") - return True - - -def source_markdown() -> list[str]: - """Return tracked and nonignored new Markdown in deterministic order.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - "*.md", - ], - check=False, - stdout=subprocess.PIPE, - ) - if completed.returncode != 0: - raise RuntimeError("git ls-files failed while selecting Markdown") - paths = [] - for raw_path in completed.stdout.split(b"\0"): - if raw_path: - path = os.fsdecode(raw_path) - if admit_source_path(path): - paths.append(path) - paths.sort() - if not paths: - raise RuntimeError("the source Markdown corpus is empty") - return paths - - -def main() -> int: - """Run deterministic Markdown and internal-link checks.""" - try: - linter = find_linter() - link_checker = find_link_checker() - verify_linter_version(linter) - verify_link_checker_version(link_checker) - paths = source_markdown() - except RuntimeError as error: - print(f"Markdown check refused: {error}", file=sys.stderr) - return 1 - - lint_result = subprocess.run( - [linter, "--no-globs", "--", *paths], - check=False, - ) - link_result = subprocess.run( - [ - link_checker, - "--offline", - "--include-fragments", - "--no-progress", - "--format", - "detailed", - "--", - *paths, - ], - check=False, - ) - return lint_result.returncode or link_result.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py deleted file mode 100644 index abb556b..0000000 --- a/scripts/check_workflows.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -"""Check every GitHub Actions workflow with the pinned actionlint tool.""" - -from __future__ import annotations - -import os -import shutil -import stat -import subprocess -import sys - -EXPECTED_VERSION = "1.7.12" - - -def find_actionlint() -> str: - """Return the actionlint path or report a precise setup failure.""" - executable = shutil.which("actionlint") - if executable is None: - raise RuntimeError("actionlint is unavailable; install version 1.7.12") - return executable - - -def verify_version(executable: str) -> None: - """Refuse an actionlint version outside the reviewed tool boundary.""" - completed = subprocess.run( - [executable, "-version"], - check=False, - capture_output=True, - text=True, - ) - first_line = completed.stdout.splitlines()[:1] - observed = first_line[0] if first_line else "" - if completed.returncode != 0 or observed != EXPECTED_VERSION: - raise RuntimeError( - f"actionlint version mismatch: " - f"expected {EXPECTED_VERSION!r}, observed {observed!r}" - ) - - -def admit_workflow_path(path: str) -> bool: - """Admit one existing regular workflow without following links.""" - try: - mode = os.lstat(path).st_mode - except FileNotFoundError: - return False - except OSError as error: - raise RuntimeError( - f"cannot inspect workflow source {path!r}: {error}" - ) from error - if not stat.S_ISREG(mode): - raise RuntimeError(f"workflow source is not a regular file: {path!r}") - return True - - -def workflow_paths() -> list[str]: - """Return Git-admitted workflow paths in deterministic order.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - ".github/workflows/*.yml", - ".github/workflows/*.yaml", - ], - check=False, - stdout=subprocess.PIPE, - ) - if completed.returncode != 0: - raise RuntimeError("git ls-files failed while selecting workflows") - paths = [] - for raw_path in completed.stdout.split(b"\0"): - if raw_path: - path = os.fsdecode(raw_path) - if admit_workflow_path(path): - paths.append(path) - paths.sort() - if not paths: - raise RuntimeError("the GitHub Actions workflow corpus is empty") - return paths - - -def main() -> int: - """Run actionlint against the deterministic workflow input boundary.""" - try: - executable = find_actionlint() - verify_version(executable) - paths = workflow_paths() - except RuntimeError as error: - print(f"Workflow check refused: {error}", file=sys.stderr) - return 1 - - completed = subprocess.run( - [executable, "-shellcheck=", "-pyflakes=", *paths], - check=False, - ) - return completed.returncode - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/test_check_markdown.py b/scripts/test_check_markdown.py deleted file mode 100644 index c5a4755..0000000 --- a/scripts/test_check_markdown.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Laws for the repository-owned Markdown source boundary.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from check_markdown import source_markdown - -REPOSITORY_ROOT = Path(__file__).parent.parent - - -class MarkdownCorpusLaws(unittest.TestCase): - """The selected Markdown corpus depends only on admitted source files.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - self.run_git("init", "--quiet") - - def run_git(self, *arguments: str) -> None: - """Run one Git setup command inside the isolated repository.""" - subprocess.run( - ["git", *arguments], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - def test_repository_ignored_markdown_cannot_enter_the_corpus(self) -> None: - self.write(".gitignore", "/target/\n") - self.write("tracked.md", "# Tracked\n") - self.write("new.md", "# New\n") - self.write("target/generated.md", "# Generated\n") - self.run_git("add", ".gitignore", "tracked.md") - - self.assertEqual(source_markdown(), ["new.md", "tracked.md"]) - - def test_source_paths_have_canonical_lexical_order(self) -> None: - self.write("zulu.md", "# Zulu\n") - self.write("alpha.md", "# Alpha\n") - self.run_git("add", "zulu.md", "alpha.md") - - self.assertEqual(source_markdown(), ["alpha.md", "zulu.md"]) - - def test_deleted_tracked_markdown_is_not_forwarded(self) -> None: - self.write("deleted.md", "# Deleted\n") - self.write("remaining.md", "# Remaining\n") - self.run_git("add", "deleted.md", "remaining.md") - (self.root / "deleted.md").unlink() - - self.assertEqual(source_markdown(), ["remaining.md"]) - - def test_user_global_ignores_cannot_change_the_corpus(self) -> None: - self.write("tracked.md", "# Tracked\n") - self.write("new.md", "# New\n") - self.write("global-ignore", "*.md\n") - self.run_git("add", "tracked.md") - global_config = self.root / "global.gitconfig" - subprocess.run( - [ - "git", - "config", - "--file", - str(global_config), - "core.excludesFile", - str(self.root / "global-ignore"), - ], - check=True, - capture_output=True, - ) - - with patch.dict( - os.environ, - { - "GIT_CONFIG_GLOBAL": str(global_config), - "GIT_CONFIG_NOSYSTEM": "1", - }, - ): - self.assertEqual(source_markdown(), ["new.md", "tracked.md"]) - - def test_symlinked_markdown_is_refused(self) -> None: - self.write("target/generated.md", "# Generated\n") - (self.root / "linked.md").symlink_to("target/generated.md") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - source_markdown() - - def test_fifo_markdown_is_refused(self) -> None: - self.write("blocking.md", "# Initially regular\n") - self.run_git("add", "blocking.md") - (self.root / "blocking.md").unlink() - os.mkfifo(self.root / "blocking.md") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - source_markdown() - - -class DocumentationCommandLaws(unittest.TestCase): - """Contributor commands inspect changes that have not been committed.""" - - def test_whitespace_checks_cover_the_index_and_working_tree(self) -> None: - for relative_path in ( - "CONTRIBUTING.md", - "docs/Documentation Standards.md", - ): - source = (REPOSITORY_ROOT / relative_path).read_text( - encoding="utf-8" - ) - self.assertNotIn( - 'git diff --check "$(git hash-object -t tree /dev/null)" HEAD', - source, - ) - self.assertIn("git diff --check\n", source) - self.assertIn("git diff --cached --check\n", source) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_dependabot_coverage.py b/scripts/test_dependabot_coverage.py deleted file mode 100644 index 0353453..0000000 --- a/scripts/test_dependabot_coverage.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Laws for complete, uniform Dependabot manifest coverage.""" - -from __future__ import annotations - -import os -import subprocess -import unittest -from pathlib import Path - -REPOSITORY_ROOT = Path(__file__).parent.parent -CONFIG_PATH = REPOSITORY_ROOT / ".github" / "dependabot.yml" -UPDATE_MARKER = " - package-ecosystem: " -MANIFEST_ECOSYSTEMS = { - "Cargo.toml": "cargo", - "package.json": "npm", -} -STATIC_SCOPES = {("github-actions", "/")} - - -def update_blocks(raw: str) -> list[str]: - """Return each Dependabot update block in source order.""" - lines = raw.splitlines() - starts = [ - index - for index, line in enumerate(lines) - if line.startswith(UPDATE_MARKER) - ] - return [ - "\n".join(lines[start:end]) - for start, end in zip(starts, [*starts[1:], len(lines)], strict=True) - ] - - -def unquote(raw: str) -> str: - """Remove one matching YAML scalar quote pair.""" - if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: - return raw[1:-1] - return raw - - -def configured_scopes(blocks: list[str]) -> list[tuple[str, str]]: - """Return every configured ecosystem and manifest directory pair.""" - scopes: list[tuple[str, str]] = [] - for block in blocks: - lines = block.splitlines() - ecosystem = unquote(lines[0].removeprefix(UPDATE_MARKER)) - for index, line in enumerate(lines): - if line.startswith(" directory: "): - scopes.append( - (ecosystem, unquote(line.removeprefix(" directory: "))) - ) - if line == " directories:": - for directory_line in lines[index + 1 :]: - if not directory_line.startswith(" - "): - break - scopes.append( - ( - ecosystem, - unquote(directory_line.removeprefix(" - ")), - ) - ) - return scopes - - -def tracked_manifest_scopes() -> set[tuple[str, str]]: - """Return dependency scopes implied by tracked first-party manifests.""" - completed = subprocess.run( - [ - "git", - "ls-files", - "-z", - "--", - ":(glob)**/Cargo.toml", - ":(glob)**/package.json", - ], - cwd=REPOSITORY_ROOT, - check=True, - stdout=subprocess.PIPE, - ) - scopes = set() - for raw_path in completed.stdout.split(b"\0"): - if not raw_path: - continue - path = Path(os.fsdecode(raw_path)) - ecosystem = MANIFEST_ECOSYSTEMS[path.name] - directory = ( - "/" - if path.parent == Path(".") - else f"/{path.parent.as_posix()}" - ) - scopes.add((ecosystem, directory)) - return scopes.union(STATIC_SCOPES) - - -class DependabotCoverageLaws(unittest.TestCase): - """Every first-party manifest receives one uniform update policy.""" - - def setUp(self) -> None: - self.raw = CONFIG_PATH.read_text(encoding="utf-8") - self.blocks = update_blocks(self.raw) - - def test_every_dependency_manifest_has_an_update_scope(self) -> None: - observed = set(configured_scopes(self.blocks)) - self.assertEqual(tracked_manifest_scopes().difference(observed), set()) - - def test_update_scopes_are_unique(self) -> None: - scopes = configured_scopes(self.blocks) - self.assertEqual(len(scopes), len(set(scopes))) - - def test_every_update_uses_the_maintenance_policy(self) -> None: - self.assertTrue(self.raw.startswith("version: 2\nupdates:\n")) - self.assertTrue(self.blocks) - for block in self.blocks: - self.assertIn(" schedule:\n interval: weekly", block) - self.assertIn(" open-pull-requests-limit: 5", block) - self.assertIn(" labels:\n - dependencies", block) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_documentation_integrity.py b/scripts/test_documentation_integrity.py deleted file mode 100644 index 200290a..0000000 --- a/scripts/test_documentation_integrity.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Negative laws for the required documentation integrity gates.""" - -from __future__ import annotations - -import json -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -import check_markdown -import check_workflows - -REPOSITORY_ROOT = Path(__file__).parent.parent -LAW_COMMAND = "python3 -m unittest discover -s scripts -p 'test_*.py' -v" - - -class ToolVersionLaws(unittest.TestCase): - """Every required checker refuses an unexpected executable version.""" - - def test_wrong_markdownlint_version_is_refused(self) -> None: - with patch.object( - check_markdown.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="markdownlint-cli2 v999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_markdown.verify_linter_version("markdownlint-cli2") - - def test_wrong_lychee_version_is_refused(self) -> None: - with patch.object( - check_markdown.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="lychee 999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_markdown.verify_link_checker_version("lychee") - - def test_wrong_actionlint_version_is_refused(self) -> None: - with patch.object( - check_workflows.subprocess, - "run", - return_value=SimpleNamespace( - returncode=0, - stdout="999.0.0\n", - ), - ): - with self.assertRaisesRegex(RuntimeError, "version mismatch"): - check_workflows.verify_version("actionlint") - - -class WorkflowContractLaws(unittest.TestCase): - """Required CI executes the repository's negative integrity laws.""" - - def test_documentation_job_runs_negative_integrity_laws(self) -> None: - workflow = ( - REPOSITORY_ROOT / ".github" / "workflows" / "ci.yml" - ).read_text(encoding="utf-8") - self.assertIn(LAW_COMMAND, workflow) - - def test_actionlint_disables_unadmitted_auxiliary_linters(self) -> None: - completed = SimpleNamespace(returncode=0) - with ( - patch.object( - check_workflows, - "find_actionlint", - return_value="actionlint", - ), - patch.object(check_workflows, "verify_version"), - patch.object( - check_workflows, - "workflow_paths", - return_value=[".github/workflows/ci.yml"], - ), - patch.object( - check_workflows.subprocess, - "run", - return_value=completed, - ) as run, - ): - self.assertEqual(check_workflows.main(), 0) - - run.assert_called_once_with( - [ - "actionlint", - "-shellcheck=", - "-pyflakes=", - ".github/workflows/ci.yml", - ], - check=False, - ) - - -class ToolInstallerLaws(unittest.TestCase): - """The Markdown tool graph is fully locked before network installation.""" - - def test_patched_parser_is_directly_admitted_without_override(self) -> None: - tool_directory = ( - REPOSITORY_ROOT / "scripts" / "documentation-tools" - ) - manifest = json.loads( - (tool_directory / "package.json").read_text(encoding="utf-8") - ) - lock = json.loads( - (tool_directory / "package-lock.json").read_text(encoding="utf-8") - ) - - self.assertNotIn("overrides", manifest) - self.assertEqual( - lock["packages"]["node_modules/markdownlint-cli2"][ - "dependencies" - ]["js-yaml"], - "5.2.2", - ) - - def test_known_parser_denial_of_service_versions_are_refused(self) -> None: - lock_path = ( - REPOSITORY_ROOT - / "scripts" - / "documentation-tools" - / "package-lock.json" - ) - packages = json.loads(lock_path.read_text(encoding="utf-8"))[ - "packages" - ] - self.assertEqual( - packages["node_modules/js-yaml"]["version"], - "5.2.2", - ) - self.assertEqual( - packages["node_modules/markdown-it"]["version"], - "14.3.0", - ) - - def test_markdown_dependency_graph_is_lockfile_admitted(self) -> None: - tool_directory = ( - REPOSITORY_ROOT / "scripts" / "documentation-tools" - ) - lock_path = tool_directory / "package-lock.json" - self.assertTrue(lock_path.is_file()) - lock = json.loads(lock_path.read_text(encoding="utf-8")) - self.assertEqual(lock["lockfileVersion"], 3) - self.assertEqual( - lock["packages"][""]["dependencies"]["markdownlint-cli2"], - "0.23.2", - ) - for path, package in lock["packages"].items(): - if path: - self.assertIn("resolved", package, path) - self.assertIn("integrity", package, path) - - installer = ( - REPOSITORY_ROOT / "scripts" / "install_documentation_tools.sh" - ).read_text(encoding="utf-8") - self.assertIn("npm ci", installer) - self.assertIn("package-lock.json", installer) - self.assertNotIn("npm install \\", installer) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_documentation_refusals.py b/scripts/test_documentation_refusals.py deleted file mode 100644 index bc357cf..0000000 --- a/scripts/test_documentation_refusals.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Precise refusal laws for malformed documentation and workflows.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path - -import check_markdown -import check_workflows - - -class IsolatedRepositoryTestCase(unittest.TestCase): - """Own one temporary Git repository for a malformed source fixture.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - subprocess.run( - ["git", "init", "--quiet"], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - -class IntegrityRefusalLaws(IsolatedRepositoryTestCase): - """Malformed inputs produce the expected checker failure class.""" - - def test_broken_internal_fragment_is_refused(self) -> None: - self.write( - "source.md", - "# Source\n\n[Missing](target.md#missing-heading)\n", - ) - self.write("target.md", "# Present heading\n") - subprocess.run( - ["git", "add", "source.md", "target.md"], - check=True, - capture_output=True, - ) - linter = check_markdown.find_linter() - link_checker = check_markdown.find_link_checker() - check_markdown.verify_linter_version(linter) - check_markdown.verify_link_checker_version(link_checker) - paths = check_markdown.source_markdown() - - lint = subprocess.run( - [linter, "--no-globs", "--", *paths], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(lint.returncode, 0, lint.stderr) - links = subprocess.run( - [ - link_checker, - "--offline", - "--include-fragments", - "--no-progress", - "--format", - "detailed", - "--", - *paths, - ], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(links.returncode, 2) - self.assertIn( - "Cannot find fragment", - f"{links.stdout}\n{links.stderr}", - ) - - def test_invalid_workflow_is_refused(self) -> None: - self.write( - ".github/workflows/invalid.yml", - "name: Invalid\non: [push\n", - ) - actionlint = check_workflows.find_actionlint() - check_workflows.verify_version(actionlint) - paths = check_workflows.workflow_paths() - - completed = subprocess.run( - [actionlint, "-shellcheck=", "-pyflakes=", *paths], - check=False, - capture_output=True, - text=True, - ) - self.assertEqual(completed.returncode, 1) - self.assertIn( - "could not parse as YAML", - f"{completed.stdout}\n{completed.stderr}", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/test_workflow_corpus.py b/scripts/test_workflow_corpus.py deleted file mode 100644 index 8531131..0000000 --- a/scripts/test_workflow_corpus.py +++ /dev/null @@ -1,147 +0,0 @@ -"""Laws for the repository-owned GitHub Actions source boundary.""" - -from __future__ import annotations - -import os -import subprocess -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from check_workflows import workflow_paths - - -class WorkflowCorpusLaws(unittest.TestCase): - """The selected workflow corpus contains only admitted source files.""" - - def setUp(self) -> None: - original_directory = Path.cwd() - self.addCleanup(os.chdir, original_directory) - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - self.root = Path(temporary_directory.name) - os.chdir(self.root) - self.run_git("init", "--quiet") - - def run_git(self, *arguments: str) -> None: - """Run one Git setup command inside the isolated repository.""" - subprocess.run( - ["git", *arguments], - check=True, - capture_output=True, - ) - - def write(self, path: str, content: str) -> None: - """Create one regular UTF-8 fixture file.""" - destination = self.root / path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content, encoding="utf-8") - - def test_repository_ignored_workflow_cannot_enter_the_corpus(self) -> None: - self.write(".gitignore", "/.github/workflows/generated.yml\n") - self.write(".github/workflows/tracked.yml", "name: Tracked\n") - self.write(".github/workflows/new.yml", "name: New\n") - self.write(".github/workflows/generated.yml", "name: Generated\n") - self.run_git( - "add", - ".gitignore", - ".github/workflows/tracked.yml", - ) - - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/new.yml", - ".github/workflows/tracked.yml", - ], - ) - - def test_source_paths_have_one_canonical_lexical_order(self) -> None: - self.write(".github/workflows/zulu.yml", "name: Zulu\n") - self.write(".github/workflows/alpha.yaml", "name: Alpha\n") - self.run_git( - "add", - ".github/workflows/zulu.yml", - ".github/workflows/alpha.yaml", - ) - - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/alpha.yaml", - ".github/workflows/zulu.yml", - ], - ) - - def test_deleted_tracked_workflow_is_not_forwarded(self) -> None: - self.write(".github/workflows/deleted.yml", "name: Deleted\n") - self.write(".github/workflows/remaining.yml", "name: Remaining\n") - self.run_git( - "add", - ".github/workflows/deleted.yml", - ".github/workflows/remaining.yml", - ) - (self.root / ".github/workflows/deleted.yml").unlink() - - self.assertEqual( - workflow_paths(), - [".github/workflows/remaining.yml"], - ) - - def test_user_global_ignores_cannot_change_the_corpus(self) -> None: - self.write(".github/workflows/tracked.yml", "name: Tracked\n") - self.write(".github/workflows/new.yml", "name: New\n") - self.write("global-ignore", "*.yml\n") - self.run_git("add", ".github/workflows/tracked.yml") - global_config = self.root / "global.gitconfig" - subprocess.run( - [ - "git", - "config", - "--file", - str(global_config), - "core.excludesFile", - str(self.root / "global-ignore"), - ], - check=True, - capture_output=True, - ) - - with patch.dict( - os.environ, - { - "GIT_CONFIG_GLOBAL": str(global_config), - "GIT_CONFIG_NOSYSTEM": "1", - }, - ): - self.assertEqual( - workflow_paths(), - [ - ".github/workflows/new.yml", - ".github/workflows/tracked.yml", - ], - ) - - def test_symlinked_workflow_is_refused(self) -> None: - self.write("generated.yml", "name: Generated\n") - workflow_dir = self.root / ".github/workflows" - workflow_dir.mkdir(parents=True) - (workflow_dir / "linked.yml").symlink_to("../../generated.yml") - - with self.assertRaisesRegex(RuntimeError, "not a regular file"): - workflow_paths() - - def test_fifo_workflow_cannot_enter_the_corpus(self) -> None: - self.write(".github/workflows/regular.yml", "name: Regular\n") - workflow_dir = self.root / ".github/workflows" - os.mkfifo(workflow_dir / "blocking.yml") - - self.assertEqual( - workflow_paths(), - [".github/workflows/regular.yml"], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 654d33f..800caaf 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -7,6 +7,7 @@ mod execution; mod node_toolchain; mod repository_text; mod tool; +mod workflow_contract; use std::path::{Path, PathBuf}; @@ -24,6 +25,7 @@ pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { verify_root(&repository_root, repository_path)?; node_toolchain::check(&repository_root)?; dependabot::check(repository_path, &repository_root)?; + workflow_contract::check(&repository_root)?; let markdown = corpus::SourceCorpus::markdown(repository_path)?; let workflows = corpus::SourceCorpus::workflow(repository_path)?; execution::run(repository_path, markdown.paths(), workflows.paths())?; diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index 7dd94da..3829839 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -150,6 +150,9 @@ impl ToolRunner for ExternalToolRunner<'_> { } } +#[cfg(test)] +#[path = "execution/external_tests.rs"] +mod external_tests; #[cfg(test)] #[path = "execution/tests.rs"] mod tests; diff --git a/xtask/src/documentation_integrity/execution/external_tests.rs b/xtask/src/documentation_integrity/execution/external_tests.rs new file mode 100644 index 0000000..b581f2a --- /dev/null +++ b/xtask/src/documentation_integrity/execution/external_tests.rs @@ -0,0 +1,79 @@ +//! This module owns pinned-tool malformed-input refusal evidence. + +use std::fs; + +use crate::test_directory::TestDirectory; + +use super::{DocumentationError, DocumentationTool, ExternalToolRunner}; + +#[test] +#[ignore = "requires pinned documentation tools installed by the documentation CI job"] +fn broken_internal_fragment_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("broken-fragment")?; + fs::write( + directory.path().join("source.md"), + "# Source\n\n[Missing](target.md#missing-heading)\n", + )?; + fs::write(directory.path().join("target.md"), "# Present heading\n")?; + let refusal = { + let mut runner = ExternalToolRunner { + repository_root: directory.path(), + }; + super::admit_version(&mut runner, DocumentationTool::Markdownlint)?; + super::admit_version(&mut runner, DocumentationTool::Lychee)?; + super::run_check( + &mut runner, + DocumentationTool::Markdownlint, + &[String::from("source.md"), String::from("target.md")], + )?; + super::run_check( + &mut runner, + DocumentationTool::Lychee, + &[String::from("source.md"), String::from("target.md")], + ) + }; + assert!(matches!( + refusal, + Err(DocumentationError::ToolFailed { + program: "lychee", + code: Some(2), + ref stdout, + ref stderr, + }) if format!("{stdout}\n{stderr}").contains("Cannot find fragment") + )); + directory.close()?; + Ok(()) +} + +#[test] +#[ignore = "requires pinned documentation tools installed by the documentation CI job"] +fn invalid_workflow_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("invalid-workflow")?; + fs::create_dir_all(directory.path().join(".github/workflows"))?; + fs::write( + directory.path().join(".github/workflows/invalid.yml"), + "name: Invalid\non: [push\n", + )?; + let refusal = { + let mut runner = ExternalToolRunner { + repository_root: directory.path(), + }; + super::admit_version(&mut runner, DocumentationTool::Actionlint)?; + super::run_check( + &mut runner, + DocumentationTool::Actionlint, + &[String::from(".github/workflows/invalid.yml")], + ) + }; + assert!(matches!( + refusal, + Err(DocumentationError::ToolFailed { + program: "actionlint", + code: Some(1), + ref stdout, + ref stderr, + }) if format!("{stdout}\n{stderr}").contains("could not parse as YAML") + )); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs new file mode 100644 index 0000000..234fe77 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -0,0 +1,61 @@ +//! This module owns the CI documentation-job execution contract. + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text; + +const CI_PATH: &str = ".github/workflows/ci.yml"; +const DOCUMENTATION_JOB: &str = " documentation:"; +const XTASK_COMMAND: &str = "run: cargo xtask documentation-integrity-check"; + +pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { + let workflow = repository_text::read(repository_root, CI_PATH)?; + admit(&workflow) +} + +fn admit(workflow: &str) -> Result<(), DocumentationError> { + let job = documentation_job(workflow)?; + if !job.contains("run: rustup show") { + return Err(contract( + "documentation job installs the pinned Rust toolchain", + )); + } + if job.matches(XTASK_COMMAND).count() != 1 { + return Err(contract( + "documentation job runs the Rust integrity command exactly once", + )); + } + if job.contains("python3") { + return Err(contract("documentation job contains no Python execution")); + } + Ok(()) +} + +fn documentation_job(workflow: &str) -> Result { + let mut lines = workflow + .lines() + .skip_while(|line| *line != DOCUMENTATION_JOB); + if lines.next().is_none() { + return Err(contract("workflow defines the documentation job")); + } + let job: Vec<_> = lines + .take_while(|line| line.starts_with(" ") || line.is_empty() || !line.starts_with(" ")) + .collect(); + if job.is_empty() { + Err(contract("documentation job is not empty")) + } else { + Ok(job.join("\n")) + } +} + +const fn contract(requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { + path: CI_PATH, + requirement, + } +} + +#[cfg(test)] +#[path = "workflow_contract/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs new file mode 100644 index 0000000..ff84708 --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -0,0 +1,47 @@ +use std::path::Path; + +use crate::repository_file::RepositoryRoot; + +const WORKFLOW: &str = r"name: CI +jobs: + documentation: + name: Documentation + steps: + - name: Install Rust + run: rustup show + - name: Verify + run: cargo xtask documentation-integrity-check + next-job: + steps: [] +"; + +#[test] +fn documentation_job_delegates_once_to_the_rust_boundary() { + assert!(super::admit(WORKFLOW).is_ok()); +} + +#[test] +fn documentation_job_refuses_python_execution() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Legacy checker\n run: python3 scripts/check_markdown.py\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job contains no Python execution", + }) + )); +} + +#[test] +fn committed_documentation_job_uses_the_rust_boundary() -> Result<(), Box> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("xtask manifest has no repository parent")?; + let repository_root = RepositoryRoot::open(root)?; + super::check(&repository_root)?; + assert!(repository_root.is_current_path()?); + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 70deeda..174057c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -122,6 +122,7 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { source_structure::check(repository_root)?; } "verify" => { + documentation_integrity::check(repository_root)?; golden_file_worldline::check(repository_root)?; protocol_conformance::check(repository_root)?; source_structure::check(repository_root)?; diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 579bace..208ea25 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -2,6 +2,7 @@ mod repository_path; mod source_error; +mod source_kind; use std::collections::BTreeSet; use std::io::{self, BufRead, BufReader}; @@ -11,9 +12,9 @@ use crate::git_inventory::{GitPath, paths as git_paths}; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; +use source_kind::{is_python_module, is_source_module}; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; -const SOURCE_SUFFIXES: [[u8; 2]; 3] = [*b"py", *b"rs", *b"sh"]; const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ "ls-files", "-z", @@ -83,13 +84,21 @@ fn select_source_paths( } fn admit_source_path(path: &GitPath) -> Result { + let python = is_python_module(path.as_bytes()); let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { SourceStructureError::GitPathEncoding { operation: "source path admission", source, } })?; - RepositoryPath::admit(text) + let relative = RepositoryPath::admit(text)?; + if python { + Err(SourceStructureError::PythonSource( + relative.as_str().to_owned(), + )) + } else { + Ok(relative) + } } fn source_violations( @@ -130,20 +139,6 @@ fn source_line_count_with( .map_err(|source| SourceStructureError::Inspect { path, source }) } -fn is_source_module(path: &[u8]) -> bool { - let Some(file_name) = path.rsplit(|byte| *byte == b'/').next() else { - return false; - }; - let mut components = file_name.rsplitn(2, |byte| *byte == b'.'); - let Some(suffix) = components.next() else { - return false; - }; - let Some(stem) = components.next() else { - return false; - }; - !stem.is_empty() && SOURCE_SUFFIXES.iter().any(|candidate| suffix == candidate) -} - const fn exceeds_hard_limit(lines: u64) -> bool { lines > SOURCE_MODULE_HARD_LIMIT_LINES } @@ -209,5 +204,8 @@ impl LineCounter { } } +#[cfg(test)] +#[path = "source_structure/pure_rust_tests.rs"] +mod pure_rust_tests; #[cfg(test)] mod tests; diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs new file mode 100644 index 0000000..538e038 --- /dev/null +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -0,0 +1,14 @@ +//! This module owns the pure-Rust source-admission regression law. + +use crate::git_inventory::GitPath; + +#[test] +fn python_source_is_refused_by_the_pure_rust_boundary() { + for path in ["scripts/check.py", "scripts/check.PY"] { + assert!(matches!( + super::admit_source_path(&GitPath::new(path.as_bytes().to_vec())), + Err(super::SourceStructureError::PythonSource(ref observed)) + if observed == path + )); + } +} diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 4fb432b..5f583ba 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -21,6 +21,7 @@ pub(crate) enum SourceStructureError { }, InvalidPath(String), NonRegular(PathBuf), + PythonSource(String), RepositoryRootChanged(PathBuf), Violations { maximum: u64, @@ -56,6 +57,11 @@ impl fmt::Display for SourceStructureError { escaped_path(formatter, path)?; formatter.write_str("`") } + Self::PythonSource(path) => { + formatter.write_str("pure Rust source boundary refuses Python module `")?; + escaped_controls(formatter, path)?; + formatter.write_str("`") + } Self::RepositoryRootChanged(path) => { formatter.write_str("repository root changed during source inspection: `")?; escaped_path(formatter, path)?; @@ -74,6 +80,7 @@ impl Error for SourceStructureError { Self::Inspect { source, .. } => Some(source), Self::InvalidPath(_) | Self::NonRegular(_) + | Self::PythonSource(_) | Self::RepositoryRootChanged(_) | Self::Violations { .. } => None, } diff --git a/xtask/src/source_structure/source_kind.rs b/xtask/src/source_structure/source_kind.rs new file mode 100644 index 0000000..021a1bc --- /dev/null +++ b/xtask/src/source_structure/source_kind.rs @@ -0,0 +1,24 @@ +//! This module owns repository source-module classification. + +const SOURCE_SUFFIXES: [[u8; 2]; 3] = [*b"py", *b"rs", *b"sh"]; + +pub(super) fn is_source_module(path: &[u8]) -> bool { + let Some(suffix) = source_suffix(path) else { + return false; + }; + SOURCE_SUFFIXES.iter().any(|candidate| { + suffix == candidate || (*candidate == *b"py" && suffix.eq_ignore_ascii_case(b"py")) + }) +} + +pub(super) fn is_python_module(path: &[u8]) -> bool { + source_suffix(path).is_some_and(|suffix| suffix.eq_ignore_ascii_case(b"py")) +} + +fn source_suffix(path: &[u8]) -> Option<&[u8]> { + let file_name = path.rsplit(|byte| *byte == b'/').next()?; + let mut components = file_name.rsplitn(2, |byte| *byte == b'.'); + let suffix = components.next()?; + let stem = components.next()?; + (!stem.is_empty()).then_some(suffix) +} diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index 74802ae..d303086 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -18,15 +18,6 @@ fn repository_tasks_require_the_committed_dependency_graph() { assert!(CARGO_CONFIGURATION.contains("xtask = \"run --quiet --locked --package xtask --\"")); } -#[test] -fn successful_verification_is_silent() -> Result<(), io::Error> { - let output = invoke(&["verify"])?; - assert!(output.status.success()); - assert!(output.stdout.is_empty()); - assert!(output.stderr.is_empty()); - Ok(()) -} - #[test] fn chunk_identity_conformance_is_repository_owned_and_silent() -> Result<(), io::Error> { let output = invoke(&["chunk-id-conformance-check"])?; diff --git a/xtask/tests/cli_contract/documentation_tools.rs b/xtask/tests/cli_contract/documentation_tools.rs new file mode 100644 index 0000000..cb86238 --- /dev/null +++ b/xtask/tests/cli_contract/documentation_tools.rs @@ -0,0 +1,126 @@ +//! This module owns hermetic fake documentation tools for CLI verification. + +#![allow( + clippy::redundant_pub_crate, + reason = "the parent integration-test module owns this private fixture" +)] + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicU64, Ordering}; + +static SEQUENCE: AtomicU64 = AtomicU64::new(0); +const MARKER_ENVIRONMENT: &str = "KEEP_TEST_TOOL_MARKERS"; + +pub(crate) struct DocumentationTools { + root: Option, +} + +impl DocumentationTools { + pub(crate) fn create() -> Result { + let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or_else(|| io::Error::other("xtask manifest has no repository parent"))?; + let parent = repository_root.join("target/cli-contract-tools"); + fs::create_dir_all(&parent)?; + let sequence = SEQUENCE.fetch_add(1, Ordering::Relaxed); + let root = parent.join(format!("{}-{sequence}", std::process::id())); + fs::create_dir(&root)?; + let tools = Self { root: Some(root) }; + fs::create_dir(tools.bin()?)?; + fs::create_dir(tools.markers()?)?; + tools.install( + "markdownlint-cli2", + "--version", + "markdownlint-cli2 v0.23.2 (markdownlint v0.41.1)", + )?; + tools.install("lychee", "--version", "lychee 0.21.0")?; + tools.install("actionlint", "-version", "1.7.12")?; + Ok(tools) + } + + pub(crate) fn invoke(&self, arguments: &[&str]) -> Result { + Command::new(env!("CARGO_BIN_EXE_xtask")) + .args(arguments) + .env("PATH", self.path_environment()?) + .env(MARKER_ENVIRONMENT, self.markers()?) + .output() + } + + pub(crate) fn require_every_tool(&self) -> Result<(), io::Error> { + for program in ["markdownlint-cli2", "lychee", "actionlint"] { + let marker = self.markers()?.join(program); + if !marker.is_file() { + return Err(io::Error::other(format!( + "documentation tool was not invoked: {program}" + ))); + } + } + Ok(()) + } + + pub(crate) fn close(mut self) -> Result<(), io::Error> { + let root = self + .root + .take() + .ok_or_else(|| io::Error::other("documentation tool directory is already closed"))?; + fs::remove_dir_all(root) + } + + fn install( + &self, + program: &str, + version_argument: &str, + version: &str, + ) -> Result<(), io::Error> { + let script = format!( + "#!/bin/sh\n\ + : > \"${{{MARKER_ENVIRONMENT}}}/{program}\"\n\ + for argument in \"$@\"; do\n\ + \x20 if [ \"$argument\" = \"{version_argument}\" ]; then\n\ + \x20 printf '%s\\n' '{version}'\n\ + \x20 exit 0\n\ + \x20 fi\n\ + done\n\ + exit 0\n" + ); + let path = self.bin()?.join(program); + fs::write(&path, script)?; + let mut permissions = fs::metadata(&path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions) + } + + fn path_environment(&self) -> Result { + let existing = env::var_os("PATH").unwrap_or_default(); + env::join_paths(std::iter::once(self.bin()?).chain(env::split_paths(&existing))) + .map_err(io::Error::other) + } + + fn bin(&self) -> Result { + Ok(self.root()?.join("bin")) + } + + fn markers(&self) -> Result { + Ok(self.root()?.join("markers")) + } + + fn root(&self) -> Result<&Path, io::Error> { + self.root + .as_deref() + .ok_or_else(|| io::Error::other("documentation tool directory is closed")) + } +} + +impl Drop for DocumentationTools { + fn drop(&mut self) { + if let Some(root) = self.root.take() { + drop(fs::remove_dir_all(root)); + } + } +} diff --git a/xtask/tests/documentation_cli_contract.rs b/xtask/tests/documentation_cli_contract.rs new file mode 100644 index 0000000..d1a1c24 --- /dev/null +++ b/xtask/tests/documentation_cli_contract.rs @@ -0,0 +1,20 @@ +//! Subprocess contract for repository-wide verification with hermetic tools. + +#![cfg(all(feature = "repository-tasks", unix))] + +#[path = "cli_contract/documentation_tools.rs"] +mod documentation_tools; + +use std::io; + +#[test] +fn successful_verification_runs_every_documentation_tool_silently() -> Result<(), io::Error> { + let tools = documentation_tools::DocumentationTools::create()?; + let output = tools.invoke(&["verify"])?; + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + assert!(output.stderr.is_empty()); + tools.require_every_tool()?; + tools.close()?; + Ok(()) +} From 5139ddb17707a355dff97c92a9c67dc49f5dcf5e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:53:39 -0700 Subject: [PATCH 013/113] Fix: admit the documentation manifest version --- .../documentation_integrity/node_toolchain.rs | 16 ++++++++++++++-- .../node_toolchain/tests.rs | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index 842c131..e1fa3ed 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -27,10 +27,22 @@ fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), Documentatio } fn admit_manifest(manifest: &Value) -> Result<(), DocumentationError> { - if manifest.get("overrides").is_none() { + if manifest.get("overrides").is_some() { + return Err(contract(MANIFEST_PATH, "dependency overrides are absent")); + } + let observed = manifest + .get("dependencies") + .and_then(|dependencies| dependencies.get("markdownlint-cli2")) + .and_then(Value::as_str); + if observed == Some("0.23.2") { Ok(()) } else { - Err(contract(MANIFEST_PATH, "dependency overrides are absent")) + Err(DocumentationError::RepositoryValue { + path: MANIFEST_PATH, + field: "dependencies.markdownlint-cli2", + expected: "0.23.2", + observed: observed.map(str::to_owned), + }) } } diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs index 9c202bb..e709818 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -43,6 +43,20 @@ fn dependency_overrides_are_refused() { )); } +#[test] +fn manifest_dependency_version_drift_is_refused() { + let manifest = r#"{"dependencies":{"markdownlint-cli2":"999.0.0"}}"#; + assert!(matches!( + super::admit(manifest, LOCK, INSTALLER), + Err(super::DocumentationError::RepositoryValue { + path: super::MANIFEST_PATH, + field: "dependencies.markdownlint-cli2", + expected: "0.23.2", + observed: Some(ref observed), + }) if observed == "999.0.0" + )); +} + #[test] fn dependency_version_drift_is_refused() { let lock = LOCK.replacen("\"5.2.2\"", "\"5.2.1\"", 1); From e991908feef066b6d706f3749a5440d2f11b0ec5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:55:58 -0700 Subject: [PATCH 014/113] Fix: refuse duplicate repository JSON fields --- Cargo.lock | 1 + docs/dependencies/serde-json-1.0.151.md | 30 ++-- xtask/Cargo.toml | 3 + .../documentation_integrity/node_toolchain.rs | 4 +- .../node_toolchain/tests.rs | 22 +++ .../node_toolchain/unique_json.rs | 130 ++++++++++++++++++ 6 files changed, 175 insertions(+), 15 deletions(-) create mode 100644 xtask/src/documentation_integrity/node_toolchain/unique_json.rs diff --git a/Cargo.lock b/Cargo.lock index aea746d..eef5ddc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -678,6 +678,7 @@ dependencies = [ "cap-fs-ext", "cap-std", "md-5", + "serde", "serde_json", ] diff --git a/docs/dependencies/serde-json-1.0.151.md b/docs/dependencies/serde-json-1.0.151.md index 5f91339..6ae61a5 100644 --- a/docs/dependencies/serde-json-1.0.151.md +++ b/docs/dependencies/serde-json-1.0.151.md @@ -8,12 +8,14 @@ ## Admitted use -Keep admits the exactly pinned `serde_json` 1.0.151 package only behind the -`xtask` crate's `repository-tasks` feature. It parses the committed Node -documentation-tool manifest and lockfile so the Rust documentation integrity -task can validate their structure and reviewed dependency versions. - -The dependency is absent from Keep's published library graph, public API, +Keep admits the exactly pinned `serde_json` 1.0.151 and `serde` 1.0.229 +packages only behind the `xtask` crate's `repository-tasks` feature. They parse +the committed Node documentation-tool manifest and lockfile so the Rust +documentation integrity task can validate their structure and reviewed +dependency versions. A Keep-owned recursive visitor rejects duplicate object +members at every depth before returning a JSON value. + +The dependencies are absent from Keep's published library graph, public API, content identities, durable formats, and production behavior. No dependency-owned type crosses out of the private repository-task adapter. @@ -31,24 +33,24 @@ capability-relative, no-follow file boundary before parsing them. ## Features and resolved graph -The direct dependency disables default features and enables only `std`. It is -optional and activated solely by `repository-tasks`. +Both direct dependencies disable default features and enable only `std`. They +are optional and activated solely by `repository-tasks`. The active normal dependency graph introduced for this boundary consists of: - `itoa` 1.0.18; - `memchr` 2.8.3; -- `serde_core` 1.0.229; and +- `serde` and `serde_core` 1.0.229; and - `zmij` 1.0.23. -Cargo's all-target resolution also retains `serde` 1.0.229, `serde_derive` -1.0.229, and `syn` 3.0.3. Their procedural-macro dependencies were already -present in the workspace lockfile. +Cargo's all-target resolution also retains `serde_derive` 1.0.229 and `syn` +3.0.3. Their procedural-macro dependencies were already present in the +workspace lockfile. ## Safety, licensing, and compatibility -`serde_json` declares the MIT OR Apache-2.0 license expression. Its manifest -declares Rust 1.71 as its minimum supported Rust version, below Keep's pinned +`serde_json` and `serde` declare the MIT OR Apache-2.0 license expression. +Their manifests declare minimum supported Rust versions below Keep's pinned toolchain. Keep-owned code invokes only safe APIs. The parser and its transitive diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 1a51a63..86ff06e 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,6 +14,7 @@ repository-tasks = [ "dep:cap-fs-ext", "dep:cap-std", "dep:md-5", + "dep:serde", "dep:serde_json", ] @@ -25,6 +26,8 @@ cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], cap-std = { version = "=4.0.2", default-features = false, optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } +# Serde drives duplicate-refusing repository JSON admission; no types escape xtask. +serde = { version = "=1.0.229", default-features = false, features = ["std"], optional = true } # Typed JSON admission checks the committed documentation-tool lock graph. serde_json = { version = "=1.0.151", default-features = false, features = ["std"], optional = true } diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index e1fa3ed..913b589 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -1,5 +1,7 @@ //! This module owns the committed Node documentation-tool graph contract. +mod unique_json; + use serde_json::{Map, Value}; use crate::repository_file::RepositoryRoot; @@ -108,7 +110,7 @@ fn admit_installer(installer: &str) -> Result<(), DocumentationError> { } fn parse(path: &'static str, raw: &str) -> Result { - serde_json::from_str(raw).map_err(|source| DocumentationError::RepositoryJson { path, source }) + unique_json::parse(raw).map_err(|source| DocumentationError::RepositoryJson { path, source }) } fn require_package_value( diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs index e709818..64a9a7a 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -57,6 +57,28 @@ fn manifest_dependency_version_drift_is_refused() { )); } +#[test] +fn duplicate_object_members_are_refused_at_every_depth() { + let manifest = concat!( + r#"{"dependencies":{"markdownlint-cli2":"999.0.0"},"#, + r#""dependencies":{"markdownlint-cli2":"0.23.2"}}"#, + ); + let lock = LOCK.replacen( + r#""version": "14.3.0","#, + r#""version": "999.0.0", "version": "14.3.0","#, + 1, + ); + for result in [ + super::admit(manifest, LOCK, INSTALLER), + super::admit(MANIFEST, &lock, INSTALLER), + ] { + assert!(matches!( + result, + Err(super::DocumentationError::RepositoryJson { .. }) + )); + } +} + #[test] fn dependency_version_drift_is_refused() { let lock = LOCK.replacen("\"5.2.2\"", "\"5.2.1\"", 1); diff --git a/xtask/src/documentation_integrity/node_toolchain/unique_json.rs b/xtask/src/documentation_integrity/node_toolchain/unique_json.rs new file mode 100644 index 0000000..35d5c47 --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/unique_json.rs @@ -0,0 +1,130 @@ +//! This module owns duplicate-refusing repository JSON admission. + +use std::fmt; + +use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Number, Value}; + +pub(super) fn parse(raw: &str) -> Result { + serde_json::from_str::(raw).map(|value| value.0) +} + +struct UniqueValue(Value); + +impl<'de> Deserialize<'de> for UniqueValue { + fn deserialize( + deserializer: DeserializerType, + ) -> Result + where + DeserializerType: Deserializer<'de>, + { + deserializer.deserialize_any(UniqueVisitor) + } +} + +struct UniqueVisitor; + +impl<'de> Visitor<'de> for UniqueVisitor { + type Value = UniqueValue; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("JSON without duplicate object members") + } + + fn visit_bool(self, value: bool) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Bool(value))) + } + + fn visit_i64(self, value: i64) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Number(Number::from(value)))) + } + + fn visit_u64(self, value: u64) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Number(Number::from(value)))) + } + + fn visit_f64(self, value: f64) -> Result + where + ErrorType: de::Error, + { + Number::from_f64(value) + .map(Value::Number) + .map(UniqueValue) + .ok_or_else(|| ErrorType::custom("JSON number is not finite")) + } + + fn visit_str(self, value: &str) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::String(value.to_owned()))) + } + + fn visit_string(self, value: String) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::String(value))) + } + + fn visit_none(self) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Null)) + } + + fn visit_unit(self) -> Result + where + ErrorType: de::Error, + { + Ok(UniqueValue(Value::Null)) + } + + fn visit_some( + self, + deserializer: DeserializerType, + ) -> Result + where + DeserializerType: Deserializer<'de>, + { + UniqueValue::deserialize(deserializer) + } + + fn visit_seq(self, mut sequence: Sequence) -> Result + where + Sequence: SeqAccess<'de>, + { + let mut values = Vec::new(); + while let Some(value) = sequence.next_element::()? { + values.push(value.0); + } + Ok(UniqueValue(Value::Array(values))) + } + + fn visit_map(self, mut object: Object) -> Result + where + Object: MapAccess<'de>, + { + let mut values = Map::new(); + while let Some(key) = object.next_key::()? { + if values.contains_key(&key) { + return Err(de::Error::custom(format!( + "duplicate object member {key:?}" + ))); + } + let value = object.next_value::()?; + values.insert(key, value.0); + } + Ok(UniqueValue(Value::Object(values))) + } +} From a3f3c09445300b1b445e4840096111a5fcae5575 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:56:57 -0700 Subject: [PATCH 015/113] Fix: preserve simultaneous documentation failures --- xtask/src/documentation_integrity/error.rs | 5 +++++ .../documentation_integrity/error/display.rs | 3 +++ .../src/documentation_integrity/execution.rs | 17 ++++++++++++++-- .../execution/tests.rs | 20 +++++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 8db0f92..e20a3b9 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -12,6 +12,10 @@ use crate::bounded_process::ProcessError; use crate::git_inventory::GitInventoryError; pub(crate) enum DocumentationError { + CheckFailures { + first: Box, + second: Box, + }, EmptyCorpus(&'static str), GitInventory(GitInventoryError), Inspect { @@ -101,6 +105,7 @@ impl fmt::Debug for DocumentationError { impl Error for DocumentationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + Self::CheckFailures { first, .. } => Some(first), Self::GitInventory(error) => Some(error), Self::Process(error) => Some(error), Self::Inspect { source, .. } | Self::RepositoryFileInspect { source, .. } => { diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 07bbcaf..99dabf9 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -22,6 +22,9 @@ enum RepositoryRootDiagnostic { impl fmt::Display for DocumentationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::CheckFailures { first, second } => { + write!(formatter, "{first}; additionally: {second}") + } Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), Self::GitInventory(error) => write!(formatter, "{error}"), Self::Inspect { corpus, path, .. } => { diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index 3829839..b53841a 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -44,12 +44,25 @@ fn run_with( admit_version(runner, DocumentationTool::Lychee)?; let lint = run_check(runner, DocumentationTool::Markdownlint, markdown); let links = run_check(runner, DocumentationTool::Lychee, markdown); - lint?; - links?; + combine_checks(lint, links)?; admit_version(runner, DocumentationTool::Actionlint)?; run_check(runner, DocumentationTool::Actionlint, workflows) } +fn combine_checks( + first: Result<(), DocumentationError>, + second: Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { + match (first, second) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(first), Err(second)) => Err(DocumentationError::CheckFailures { + first: Box::new(first), + second: Box::new(second), + }), + } +} + fn admit_version( runner: &mut impl ToolRunner, tool: DocumentationTool, diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index 5e421ca..a55b55c 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -75,6 +75,26 @@ fn link_check_runs_after_markdownlint_returns_nonzero() { ); } +#[test] +fn simultaneous_markdown_failures_are_both_reported() -> Result<(), &'static str> { + let mut runner = RecordingRunner::new([ + version(DocumentationTool::Markdownlint), + version(DocumentationTool::Lychee), + failure(b"lint refusal", b""), + failure(b"link refusal", b""), + ]); + + let result = super::run_with(&mut runner, &[String::from("README.md")], &[]); + let diagnostic = result + .err() + .ok_or("both Markdown checks unexpectedly succeeded")? + .to_string(); + + assert!(diagnostic.contains("lint refusal")); + assert!(diagnostic.contains("link refusal")); + Ok(()) +} + #[test] fn unreviewed_version_stops_before_tool_execution() { let mut runner = RecordingRunner::new([ProcessOutput { From 0d5895dcc66159a9bc09fc02025fb114ac7e87b0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 12:58:54 -0700 Subject: [PATCH 016/113] Fix: port contributor whitespace command laws --- xtask/src/documentation_integrity.rs | 2 + .../contributor_contract.rs | 86 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 xtask/src/documentation_integrity/contributor_contract.rs diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 800caaf..d0308a4 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -1,5 +1,6 @@ //! This module owns documentation and workflow integrity orchestration. +mod contributor_contract; mod corpus; mod dependabot; mod error; @@ -23,6 +24,7 @@ pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { } })?; verify_root(&repository_root, repository_path)?; + contributor_contract::check(&repository_root)?; node_toolchain::check(&repository_root)?; dependabot::check(repository_path, &repository_root)?; workflow_contract::check(&repository_root)?; diff --git a/xtask/src/documentation_integrity/contributor_contract.rs b/xtask/src/documentation_integrity/contributor_contract.rs new file mode 100644 index 0000000..e41871e --- /dev/null +++ b/xtask/src/documentation_integrity/contributor_contract.rs @@ -0,0 +1,86 @@ +//! This module owns contributor-facing documentation command contracts. + +use crate::repository_file::RepositoryRoot; + +use super::error::DocumentationError; +use super::repository_text; + +const CONTRIBUTING_PATH: &str = "CONTRIBUTING.md"; +const STANDARDS_PATH: &str = "docs/Documentation Standards.md"; +const UNSTAGED_CHECK: &str = "git diff --check"; +const STAGED_CHECK: &str = "git diff --cached --check"; +const WHOLE_TREE_CHECK: &str = r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#; + +pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { + for path in [CONTRIBUTING_PATH, STANDARDS_PATH] { + let raw = repository_text::read(repository_root, path)?; + admit(path, &raw)?; + } + Ok(()) +} + +fn admit(path: &'static str, raw: &str) -> Result<(), DocumentationError> { + if raw.lines().any(|line| line == WHOLE_TREE_CHECK) { + return Err(contract( + path, + "contributor command does not replace change checks with a whole-tree check", + )); + } + require_line( + path, + raw, + UNSTAGED_CHECK, + "documents the unstaged whitespace check", + )?; + require_line( + path, + raw, + STAGED_CHECK, + "documents the staged whitespace check", + ) +} + +fn require_line( + path: &'static str, + raw: &str, + expected: &str, + requirement: &'static str, +) -> Result<(), DocumentationError> { + if raw.lines().any(|line| line == expected) { + Ok(()) + } else { + Err(contract(path, requirement)) + } +} + +const fn contract(path: &'static str, requirement: &'static str) -> DocumentationError { + DocumentationError::RepositoryContract { path, requirement } +} + +#[cfg(test)] +mod tests { + #[test] + fn whole_tree_whitespace_replacement_is_refused() { + let invalid = concat!( + "git diff --check \"$(git hash-object -t tree /dev/null)\" HEAD\n", + "git diff --check\n", + "git diff --cached --check\n", + ); + + assert!(super::admit("guide.md", invalid).is_err()); + } + + #[test] + fn contributor_commands_cover_staged_and_unstaged_whitespace() { + for invalid in ["git diff --check\n", "git diff --cached --check\n"] { + assert!(super::admit("guide.md", invalid).is_err()); + } + } + + #[test] + fn separate_change_checks_are_admitted() { + let valid = "git diff --check\ngit diff --cached --check\n"; + + assert!(super::admit("guide.md", valid).is_ok()); + } +} From dcc2e45bb28e22cf4990fb337e338298068763e3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:02:42 -0700 Subject: [PATCH 017/113] Fix: bound process output collection by deadline --- xtask/src/bounded_process.rs | 75 +++++++++--------------- xtask/src/bounded_process/deadline.rs | 44 ++++++++++++++ xtask/src/bounded_process/reader.rs | 82 +++++++++++++++++++++++++++ xtask/src/bounded_process/tests.rs | 63 ++++++++++++++++++++ 4 files changed, 215 insertions(+), 49 deletions(-) create mode 100644 xtask/src/bounded_process/deadline.rs create mode 100644 xtask/src/bounded_process/reader.rs diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 21dfb53..2d408b8 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -1,15 +1,19 @@ //! This module owns bounded external child-process collection. +mod deadline; mod error; +mod reader; use std::io; use std::process::{Child, Command, ExitStatus, Stdio}; -use std::thread::{self, JoinHandle}; +use std::thread; use std::time::{Duration, Instant}; +use deadline::ProcessDeadline; pub(crate) use error::ProcessError; +use reader::ReaderWorker; -use crate::process_output::{BoundedBytes, bounded_bytes}; +use crate::process_output::BoundedBytes; const OUTPUT_LIMIT: usize = 1_048_576; @@ -42,6 +46,7 @@ pub(crate) fn capture( command: &mut Command, deadline: Option, ) -> Result { + let deadline = ProcessDeadline::new(program, deadline)?; command.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = command.spawn().map_err(|source| ProcessError::Io { program, @@ -60,24 +65,21 @@ pub(crate) fn capture( (Ok(stdout), Ok(stderr)) => (stdout, stderr), (Err(error), _) | (_, Err(error)) => return Err(cleanup(&mut child, error)), }; - let stdout_reader = match start_reader(program, "stdout", stdout) { + let stdout_reader = match ReaderWorker::start(program, "stdout", stdout, OUTPUT_LIMIT) { Ok(reader) => reader, Err(error) => return Err(cleanup(&mut child, error)), }; - let stderr_reader = match start_reader(program, "stderr", stderr) { + let stderr_reader = match ReaderWorker::start(program, "stderr", stderr, OUTPUT_LIMIT) { Ok(reader) => reader, Err(error) => { let error = cleanup(&mut child, error); - drop(join_reader(program, "stdout", stdout_reader)); + drop(stdout_reader); return Err(error); } }; - let status = wait_for_child(program, &mut child, deadline); - let stdout = join_reader(program, "stdout", stdout_reader); - let stderr = join_reader(program, "stderr", stderr_reader); - let status = status?; - let stdout = stdout?; - let stderr = stderr?; + let status = wait_for_child(program, &mut child, &deadline)?; + let stdout = stdout_reader.collect(&deadline)?; + let stderr = stderr_reader.collect(&deadline)?; refuse_exceeded(program, "stdout", &stdout)?; refuse_exceeded(program, "stderr", &stderr)?; Ok(ProcessOutput { @@ -91,9 +93,9 @@ pub(crate) fn capture( fn wait_for_child( program: &'static str, child: &mut Child, - deadline: Option, + deadline: &ProcessDeadline, ) -> Result { - let Some(duration) = deadline else { + let ProcessDeadline::Bounded { duration, expires } = deadline else { return match child.wait() { Ok(status) => Ok(status), Err(source) => Err(cleanup( @@ -106,9 +108,6 @@ fn wait_for_child( )), }; }; - let Some(expires) = Instant::now().checked_add(duration) else { - return Err(cleanup(child, ProcessError::Timeout { program, duration })); - }; loop { match child.try_wait() { Err(source) => { @@ -122,44 +121,22 @@ fn wait_for_child( )); } Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() >= expires => { - return Err(cleanup(child, ProcessError::Timeout { program, duration })); + Ok(None) if Instant::now() >= *expires => { + return Err(cleanup( + child, + ProcessError::Timeout { + program, + duration: *duration, + }, + )); } - Ok(None) => thread::sleep(Duration::from_millis(10)), + Ok(None) => thread::sleep( + Duration::from_millis(10).min(expires.saturating_duration_since(Instant::now())), + ), } } } -fn start_reader( - program: &'static str, - stream: &'static str, - reader: impl io::Read + Send + 'static, -) -> Result>, ProcessError> { - thread::Builder::new() - .name(format!("xtask-{stream}-reader")) - .spawn(move || bounded_bytes(reader, OUTPUT_LIMIT)) - .map_err(|source| ProcessError::Io { - program, - action: "start output reader", - source, - }) -} - -fn join_reader( - program: &'static str, - stream: &'static str, - worker: JoinHandle>, -) -> Result { - worker - .join() - .map_err(|_panic| ProcessError::ReaderPanic { program, stream })? - .map_err(|source| ProcessError::Io { - program, - action: "read child output", - source, - }) -} - const fn refuse_exceeded( program: &'static str, stream: &'static str, diff --git a/xtask/src/bounded_process/deadline.rs b/xtask/src/bounded_process/deadline.rs new file mode 100644 index 0000000..3e12afe --- /dev/null +++ b/xtask/src/bounded_process/deadline.rs @@ -0,0 +1,44 @@ +//! This module owns one deadline spanning process and output collection. + +use std::time::{Duration, Instant}; + +use super::ProcessError; + +pub(super) enum ProcessDeadline { + Bounded { + duration: Duration, + expires: Instant, + }, + Unbounded, +} + +impl ProcessDeadline { + pub(super) fn new( + program: &'static str, + duration: Option, + ) -> Result { + let Some(duration) = duration else { + return Ok(Self::Unbounded); + }; + let expires = Instant::now() + .checked_add(duration) + .ok_or(ProcessError::Timeout { program, duration })?; + Ok(Self::Bounded { duration, expires }) + } + + pub(super) fn remaining( + &self, + program: &'static str, + ) -> Result, ProcessError> { + match self { + Self::Unbounded => Ok(None), + Self::Bounded { duration, expires } => expires + .checked_duration_since(Instant::now()) + .map(|remaining| Some((remaining, *duration))) + .ok_or(ProcessError::Timeout { + program, + duration: *duration, + }), + } + } +} diff --git a/xtask/src/bounded_process/reader.rs b/xtask/src/bounded_process/reader.rs new file mode 100644 index 0000000..f373411 --- /dev/null +++ b/xtask/src/bounded_process/reader.rs @@ -0,0 +1,82 @@ +//! This module owns deadline-bounded child-output collection. + +use std::io; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError}; +use std::thread::{self, JoinHandle}; + +use crate::process_output::{BoundedBytes, bounded_bytes}; + +use super::{ProcessDeadline, ProcessError}; + +pub(super) struct ReaderWorker { + handle: JoinHandle<()>, + program: &'static str, + receiver: Receiver>, + stream: &'static str, +} + +impl ReaderWorker { + pub(super) fn start( + program: &'static str, + stream: &'static str, + reader: impl io::Read + Send + 'static, + maximum: usize, + ) -> Result { + let (sender, receiver) = mpsc::sync_channel(1); + let handle = thread::Builder::new() + .name(format!("xtask-{stream}-reader")) + .spawn(move || { + drop(sender.send(bounded_bytes(reader, maximum))); + }) + .map_err(|source| ProcessError::Io { + program, + action: "start output reader", + source, + })?; + Ok(Self { + handle, + program, + receiver, + stream, + }) + } + + pub(super) fn collect(self, deadline: &ProcessDeadline) -> Result { + let Self { + handle, + program, + receiver, + stream, + } = self; + let result = match deadline.remaining(program)? { + Some((remaining, duration)) => receiver + .recv_timeout(remaining) + .map_err(|error| receive_error(program, stream, duration, error))?, + None => receiver.recv().map_err(|_| reader_panic(program, stream))?, + }; + handle + .join() + .map_err(|_panic| reader_panic(program, stream))?; + result.map_err(|source| ProcessError::Io { + program, + action: "read child output", + source, + }) + } +} + +const fn receive_error( + program: &'static str, + stream: &'static str, + duration: std::time::Duration, + error: RecvTimeoutError, +) -> ProcessError { + match error { + RecvTimeoutError::Timeout => ProcessError::Timeout { program, duration }, + RecvTimeoutError::Disconnected => reader_panic(program, stream), + } +} + +const fn reader_panic(program: &'static str, stream: &'static str) -> ProcessError { + ProcessError::ReaderPanic { program, stream } +} diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index 4a419ae..ddae442 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -3,10 +3,13 @@ use std::env; use std::io::{self, Write}; use std::process::{Command, Stdio}; +use std::sync::mpsc::{self, RecvTimeoutError}; use std::time::Duration; use super::{ProcessError, capture}; +const DESCENDANT_CHILD: &str = "KEEP_XTASK_DESCENDANT_CHILD"; +const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; const OUTPUT_CHILD: &str = "KEEP_XTASK_BOUNDED_OUTPUT_CHILD"; #[test] @@ -45,3 +48,63 @@ fn process_child_writes_excess_output() -> Result<(), io::Error> { output.write_all(&bytes)?; output.flush() } + +#[test] +fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> +{ + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::tests::process_child_leaves_descendant_pipe_open", + ]) + .env(DESCENDANT_PARENT, "1") + .stdin(Stdio::null()); + + let result = capture( + "test process", + &mut command, + Some(Duration::from_millis(25)), + ); + + assert!(matches!( + result, + Err(ProcessError::Timeout { + program: "test process", + duration, + }) if duration == Duration::from_millis(25) + )); + Ok(()) +} + +#[test] +fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { + if env::var_os(DESCENDANT_PARENT).is_none() { + return Ok(()); + } + let executable = env::current_exe()?; + drop( + Command::new(executable) + .args([ + "--exact", + "bounded_process::tests::process_descendant_holds_pipe", + ]) + .env(DESCENDANT_CHILD, "1") + .spawn()?, + ); + Ok(()) +} + +#[test] +fn process_descendant_holds_pipe() { + if env::var_os(DESCENDANT_CHILD).is_none() { + return; + } + let (sender, receiver) = mpsc::channel::<()>(); + assert!(matches!( + receiver.recv_timeout(Duration::from_millis(250)), + Err(RecvTimeoutError::Timeout) + )); + drop(sender); +} From db31c83f7cbb604255068c59c401f1b7be07a604 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:04:27 -0700 Subject: [PATCH 018/113] Fix: bind the documentation installer command --- .../documentation_integrity/node_toolchain.rs | 11 ++++++--- .../node_toolchain/tests.rs | 23 +++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index 913b589..debc6d6 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -12,6 +12,8 @@ use super::repository_text; const INSTALLER_PATH: &str = "scripts/install_documentation_tools.sh"; const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; +const NPM_CI_COMMAND: &str = + "npm ci \\\n --prefix \"$npm_dir\" \\\n --ignore-scripts \\\n --no-audit \\\n --no-fund"; pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; @@ -91,8 +93,11 @@ fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { } fn admit_installer(installer: &str) -> Result<(), DocumentationError> { - if !installer.contains("npm ci") { - return Err(contract(INSTALLER_PATH, "installation uses npm ci")); + if !installer.contains(NPM_CI_COMMAND) { + return Err(contract( + INSTALLER_PATH, + "installation uses the reviewed npm ci command", + )); } if !installer.contains("package-lock.json") { return Err(contract( @@ -100,7 +105,7 @@ fn admit_installer(installer: &str) -> Result<(), DocumentationError> { "installation requires package-lock.json", )); } - if installer.contains("npm install \\") { + if installer.contains("npm install") { return Err(contract( INSTALLER_PATH, "installation does not bypass the lock with npm install", diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs index 64a9a7a..d45e0de 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -24,7 +24,14 @@ const LOCK: &str = r#"{ } } }"#; -const INSTALLER: &str = "test -f package-lock.json\nnpm ci\n"; +const INSTALLER: &str = concat!( + "test -f package-lock.json\n", + "npm ci \\\n", + " --prefix \"$npm_dir\" \\\n", + " --ignore-scripts \\\n", + " --no-audit \\\n", + " --no-fund\n", +); #[test] fn admitted_node_toolchain_is_exact_and_lockfile_installed() { @@ -138,7 +145,19 @@ fn unlocked_installer_is_refused() { super::admit(MANIFEST, LOCK, "npm install markdownlint-cli2\n"), Err(super::DocumentationError::RepositoryContract { path: super::INSTALLER_PATH, - requirement: "installation uses npm ci", + requirement: "installation uses the reviewed npm ci command", + }) + )); +} + +#[test] +fn comments_cannot_mask_an_unlocked_installer() { + let installer = "# npm ci\n# package-lock.json\nnpm install markdownlint-cli2\n"; + assert!(matches!( + super::admit(MANIFEST, LOCK, installer), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installation uses the reviewed npm ci command", }) )); } From 146c419826cf77ee3e99cfb6e54de9a1bd1012c2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:05:45 -0700 Subject: [PATCH 019/113] Document: align documentation integrity contracts --- CHANGELOG.md | 5 ++++- docs/Documentation Standards.md | 1 + docs/dependencies/documentation-toolchain.md | 17 ++++++++++------- ... => serde-and-serde-json-1.0.229-1.0.151.md} | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) rename docs/dependencies/{serde-json-1.0.151.md => serde-and-serde-json-1.0.229-1.0.151.md} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb09e36..4c5434f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,10 @@ after its public API and format compatibility policies are established. - Documentation corpus selection, pinned tool admission, Markdown and fragment checks, workflow linting, Dependabot coverage, and Node lock-graph policy now run through bounded Rust `xtask` code; CI and `cargo xtask verify` use that - boundary, and the seven superseded Python checkers have been removed. + boundary, and the seven superseded Python checkers have been removed. The + boundary rejects duplicate repository JSON fields and unlocked installer + substitutions, retains simultaneous Markdown and link failures, and applies + one deadline across child execution and output collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index ac32c8a..f36ea26 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -52,6 +52,7 @@ Keep keeps its durable truth in a small set of known places. | `docs/recovery/` | The crash-point catalog, the publication protocol, and the documented lawful state recovery must reach. | | `docs/adr/` | Slugged ADRs for decisions that cut across subsystems or predate a colocated home. See §3.10. | | `docs///rationale.md` | Colocated decision record for a decision scoped to one concept: the decision, alternatives rejected, and why. See §3.10. | +| `docs/dependencies/` | Dependency and development-tool admission records: scope, selected features, resolved graph, risks, and review triggers. | | `CONTRIBUTING.md` | Contributor-facing operational contract: what to read, required local checks, and PR expectations. | | `CHANGELOG.md` | Release-visible historical ledger. | | `SECURITY.md` | Vulnerability reporting and support posture. | diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index e0991eb..f69eea0 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -31,18 +31,21 @@ before extraction: | `actionlint_1.7.12_linux_amd64.tar.gz` | `8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8` | | `lychee-x86_64-unknown-linux-gnu.tar.gz` | `a06547250f10021dcafc6ed5bb20fca75835b65711745b63cfdda34c29ff6a73` | -The repository checkers then verify each executable's reported version before -admitting its output as evidence. A missing tool, changed archive, unexpected -version, empty input corpus, or tool failure refuses the check. +The Rust `cargo xtask documentation-integrity-check` boundary verifies the +committed Node graph and installer command, then verifies each executable's +reported version before admitting its output as evidence. A missing tool, +changed archive, unexpected version, empty input corpus, or tool failure +refuses the check. ## Determinism and network posture -The Markdown and workflow checkers derive their inputs from Git's tracked and +The Rust task derives the Markdown and workflow inputs from Git's tracked and repository-nonignored source paths. Generated Rustdoc, build outputs, ignored fuzz artifacts, ignored vendor trees, and user-global ignore policy cannot -enter either input set. Missing tracked paths are treated as pending deletions; -Git-trackable nonregular paths such as symlinks are refused. Non-trackable -special files such as FIFOs cannot enter the Git-selected corpus. +enter either input set. Missing tracked paths are treated as pending +deletions; Git-trackable nonregular paths such as symlinks and tracked paths +replaced by FIFOs are refused. Non-trackable special files cannot enter the +Git-selected corpus. The workflow checker disables `actionlint`'s optional `shellcheck` and `pyflakes` integrations. Neither auxiliary executable is admitted or pinned by diff --git a/docs/dependencies/serde-json-1.0.151.md b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md similarity index 98% rename from docs/dependencies/serde-json-1.0.151.md rename to docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md index 6ae61a5..a019a7a 100644 --- a/docs/dependencies/serde-json-1.0.151.md +++ b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md @@ -1,4 +1,4 @@ -# Dependency Admission: serde_json 1.0.151 +# Dependency Admission: serde 1.0.229 and serde_json 1.0.151 - Status: Accepted for repository-task JSON admission only - Date: 2026-07-28 From b8c715710f9719224e5cb67058e076f1198cb7d8 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:21:42 -0700 Subject: [PATCH 020/113] Fix: bind installer admission to exact bytes --- docs/dependencies/documentation-toolchain.md | 8 ++--- .../documentation_integrity/node_toolchain.rs | 29 +++++++------------ .../node_toolchain/tests.rs | 28 +++++++++++++++--- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index f69eea0..f04803b 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -32,10 +32,10 @@ before extraction: | `lychee-x86_64-unknown-linux-gnu.tar.gz` | `a06547250f10021dcafc6ed5bb20fca75835b65711745b63cfdda34c29ff6a73` | The Rust `cargo xtask documentation-integrity-check` boundary verifies the -committed Node graph and installer command, then verifies each executable's -reported version before admitting its output as evidence. A missing tool, -changed archive, unexpected version, empty input corpus, or tool failure -refuses the check. +committed Node graph and the exact BLAKE3 digest of the reviewed installer, +then verifies each executable's reported version before admitting its output +as evidence. A missing tool, changed archive, unexpected version, empty input +corpus, or tool failure refuses the check. ## Determinism and network posture diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index debc6d6..bb99480 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -10,10 +10,12 @@ use super::error::DocumentationError; use super::repository_text; const INSTALLER_PATH: &str = "scripts/install_documentation_tools.sh"; +const INSTALLER_DIGEST: [u8; 32] = [ + 0x12, 0xfb, 0x82, 0xcd, 0xdb, 0x65, 0x52, 0xe5, 0xae, 0xdb, 0x14, 0x54, 0x83, 0xf4, 0x8a, 0x8a, + 0x8b, 0x35, 0x54, 0xea, 0x2a, 0x90, 0xeb, 0xd4, 0x82, 0xd7, 0x4a, 0x61, 0x1f, 0xf6, 0xd3, 0xfd, +]; const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; -const NPM_CI_COMMAND: &str = - "npm ci \\\n --prefix \"$npm_dir\" \\\n --ignore-scripts \\\n --no-audit \\\n --no-fund"; pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; @@ -93,25 +95,14 @@ fn admit_lock(lock: &Value) -> Result<(), DocumentationError> { } fn admit_installer(installer: &str) -> Result<(), DocumentationError> { - if !installer.contains(NPM_CI_COMMAND) { - return Err(contract( - INSTALLER_PATH, - "installation uses the reviewed npm ci command", - )); - } - if !installer.contains("package-lock.json") { - return Err(contract( - INSTALLER_PATH, - "installation requires package-lock.json", - )); - } - if installer.contains("npm install") { - return Err(contract( + if blake3::hash(installer.as_bytes()).as_bytes() == &INSTALLER_DIGEST { + Ok(()) + } else { + Err(contract( INSTALLER_PATH, - "installation does not bypass the lock with npm install", - )); + "installer bytes match the reviewed digest", + )) } - Ok(()) } fn parse(path: &'static str, raw: &str) -> Result { diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs index d45e0de..e672f05 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -24,8 +24,8 @@ const LOCK: &str = r#"{ } } }"#; -const INSTALLER: &str = concat!( - "test -f package-lock.json\n", +const INSTALLER: &str = include_str!("../../../../scripts/install_documentation_tools.sh"); +const REVIEWED_NPM_CI: &str = concat!( "npm ci \\\n", " --prefix \"$npm_dir\" \\\n", " --ignore-scripts \\\n", @@ -145,7 +145,7 @@ fn unlocked_installer_is_refused() { super::admit(MANIFEST, LOCK, "npm install markdownlint-cli2\n"), Err(super::DocumentationError::RepositoryContract { path: super::INSTALLER_PATH, - requirement: "installation uses the reviewed npm ci command", + requirement: "installer bytes match the reviewed digest", }) )); } @@ -157,7 +157,27 @@ fn comments_cannot_mask_an_unlocked_installer() { super::admit(MANIFEST, LOCK, installer), Err(super::DocumentationError::RepositoryContract { path: super::INSTALLER_PATH, - requirement: "installation uses the reviewed npm ci command", + requirement: "installer bytes match the reviewed digest", + }) + )); +} + +#[test] +fn dead_shell_structure_cannot_mask_an_unreviewed_command() { + let installer = format!( + "# package-lock.json\n\ + unused() {{\n\ + \x20 cat <<'REVIEWED'\n\ + {REVIEWED_NPM_CI}\n\ + REVIEWED\n\ + }}\n\ + npm ci --prefix /tmp/unreviewed --foreground-scripts\n", + ); + assert!(matches!( + super::admit(MANIFEST, LOCK, &installer), + Err(super::DocumentationError::RepositoryContract { + path: super::INSTALLER_PATH, + requirement: "installer bytes match the reviewed digest", }) )); } From ee3a18a04ca3ec3d484403f43899208f6caef975 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:37:40 -0700 Subject: [PATCH 021/113] Fix: terminate failed subprocess groups --- Cargo.lock | 1 + .../cap-std-and-cap-fs-ext-4.0.2.md | 47 +++-- xtask/Cargo.toml | 3 + xtask/src/bounded_process.rs | 147 +-------------- xtask/src/bounded_process/capture.rs | 174 ++++++++++++++++++ xtask/src/bounded_process/cleanup.rs | 56 ++++++ xtask/src/bounded_process/error.rs | 18 +- xtask/src/bounded_process/process_group.rs | 60 ++++++ .../process_group/child_tests.rs | 70 +++++++ .../bounded_process/process_group/tests.rs | 122 ++++++++++++ xtask/src/bounded_process/reader.rs | 31 ++-- xtask/src/bounded_process/tests.rs | 63 ------- 12 files changed, 548 insertions(+), 244 deletions(-) create mode 100644 xtask/src/bounded_process/capture.rs create mode 100644 xtask/src/bounded_process/cleanup.rs create mode 100644 xtask/src/bounded_process/process_group.rs create mode 100644 xtask/src/bounded_process/process_group/child_tests.rs create mode 100644 xtask/src/bounded_process/process_group/tests.rs diff --git a/Cargo.lock b/Cargo.lock index eef5ddc..4e671a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -678,6 +678,7 @@ dependencies = [ "cap-fs-ext", "cap-std", "md-5", + "rustix", "serde", "serde_json", ] diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 429d34c..181bfce 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -1,4 +1,4 @@ -# Dependency Admission: cap-std and cap-fs-ext 4.0.2 +# Dependency Admission: cap-std, cap-fs-ext 4.0.2, and rustix 1.1.4 - Status: Accepted for repository-task filesystem boundaries only - Date: 2026-07-26 @@ -8,8 +8,9 @@ ## Admitted use -Keep admits the exactly pinned `cap-std` 4.0.2 and `cap-fs-ext` 4.0.2 packages -only behind the `xtask` crate's `repository-tasks` feature. +Keep admits the exactly pinned `cap-std` 4.0.2, `cap-fs-ext` 4.0.2, and +`rustix` 1.1.4 packages only behind the `xtask` crate's `repository-tasks` +feature. `cap-std::fs::Dir` pins the admitted repository or corpus directory and opens entries relative to that capability. `cap-fs-ext` supplies no-follow and @@ -24,6 +25,11 @@ These packages are absent from Keep's published library graph, public API, content identities, durable formats, and production behavior. No dependency-owned type crosses out of the private repository-task adapter. +The bounded subprocess adapter uses Rustix's safe process API to send +`SIGKILL` to a dedicated child process group after a subprocess deadline or +collection failure. This prevents descendants that inherited an output pipe +from surviving the failed repository task. + ## Why the standard library is insufficient Checking a path and then reopening it with `std::fs` leaves a @@ -40,9 +46,10 @@ otherwise forbids. ## Features and resolved graph -Both direct dependencies disable default features. Keep enables only -`cap-fs-ext`'s `std` feature; `cap-std` has no enabled feature. Both declarations -are optional and are activated solely by `repository-tasks`. +All three direct dependencies disable default features. Keep enables only +`cap-fs-ext`'s `std` feature and Rustix's `process` and `std` features; +`cap-std` has no enabled feature. All declarations are optional and are +activated solely by `repository-tasks`. The locked non-Windows graph introduced for this boundary is: @@ -66,18 +73,20 @@ Windows resolution additionally retains the locked `windows-sys`, ## Safety, licensing, and compatibility -Both direct packages declare +The capability packages declare `Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT`; Keep selects an admitted -license through repository policy. Their manifests declare no Rust-version -floor. Compatibility is therefore established only by Keep's pinned stable, -MSRV, debug, release, Clippy, dependency-policy, and advisory lanes. - -The capability packages and their platform dependencies may contain unsafe -code around operating-system calls and handles. Keep-owned code invokes only -their safe APIs, retains handles in private adapter types, checks resulting -metadata, bounds reads, and never treats the dependency as proof of content -identity. `cargo deny` and RustSec checks remain mandatory point-in-time -evidence; they do not transfer Keep's unsafe-code guarantee to dependencies. +license through repository policy. Rustix declares `Apache-2.0 OR MIT`. +Their manifests declare no Rust-version floor. Compatibility is therefore +established only by Keep's pinned stable, MSRV, debug, release, Clippy, +dependency-policy, and advisory lanes. + +The admitted packages and their platform dependencies may contain unsafe code +around operating-system calls and handles. Keep-owned code invokes only their +safe APIs, retains handles and process identifiers in private adapter types, +checks resulting metadata, bounds reads, and never treats a dependency as +proof of content identity. `cargo deny` and RustSec checks remain mandatory +point-in-time evidence; they do not transfer Keep's unsafe-code guarantee to +dependencies. ## Failure and recovery boundaries @@ -87,8 +96,8 @@ verification process and carry no durability or recovery semantics. Keep can remove these dependencies without changing public or durable behavior by replacing them with an equally portable, safe implementation that preserves -capability-relative, no-follow, nonblocking, regular-file, and retained-handle -tests on every supported platform. +capability-relative, no-follow, nonblocking, regular-file, retained-handle, and +whole-process-group cleanup tests on every supported platform. Reopen this admission if either direct version, selected feature, resolved graph, license, supported platform, handle-retention invariant, or diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 86ff06e..36f7475 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,6 +14,7 @@ repository-tasks = [ "dep:cap-fs-ext", "dep:cap-std", "dep:md-5", + "dep:rustix", "dep:serde", "dep:serde_json", ] @@ -26,6 +27,8 @@ cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], cap-std = { version = "=4.0.2", default-features = false, optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } +# Safe POSIX process-group signaling bounds failed repository-tool subprocesses. +rustix = { version = "=1.1.4", default-features = false, features = ["process", "std"], optional = true } # Serde drives duplicate-refusing repository JSON admission; no types escape xtask. serde = { version = "=1.0.229", default-features = false, features = ["std"], optional = true } # Typed JSON admission checks the committed documentation-tool lock graph. diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 2d408b8..4a73e37 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -1,22 +1,19 @@ //! This module owns bounded external child-process collection. +mod capture; +mod cleanup; mod deadline; mod error; +mod process_group; mod reader; -use std::io; -use std::process::{Child, Command, ExitStatus, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; +use std::process::Command; +pub(crate) use capture::capture; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use reader::ReaderWorker; -use crate::process_output::BoundedBytes; - -const OUTPUT_LIMIT: usize = 1_048_576; - pub(crate) struct ProcessOutput { pub(crate) code: Option, pub(crate) succeeded: bool, @@ -41,140 +38,6 @@ pub(crate) fn status( }) } -pub(crate) fn capture( - program: &'static str, - command: &mut Command, - deadline: Option, -) -> Result { - let deadline = ProcessDeadline::new(program, deadline)?; - command.stdout(Stdio::piped()).stderr(Stdio::piped()); - let mut child = command.spawn().map_err(|source| ProcessError::Io { - program, - action: "spawn", - source, - })?; - let stdout = child.stdout.take().ok_or(ProcessError::MissingStream { - program, - stream: "stdout", - }); - let stderr = child.stderr.take().ok_or(ProcessError::MissingStream { - program, - stream: "stderr", - }); - let (stdout, stderr) = match (stdout, stderr) { - (Ok(stdout), Ok(stderr)) => (stdout, stderr), - (Err(error), _) | (_, Err(error)) => return Err(cleanup(&mut child, error)), - }; - let stdout_reader = match ReaderWorker::start(program, "stdout", stdout, OUTPUT_LIMIT) { - Ok(reader) => reader, - Err(error) => return Err(cleanup(&mut child, error)), - }; - let stderr_reader = match ReaderWorker::start(program, "stderr", stderr, OUTPUT_LIMIT) { - Ok(reader) => reader, - Err(error) => { - let error = cleanup(&mut child, error); - drop(stdout_reader); - return Err(error); - } - }; - let status = wait_for_child(program, &mut child, &deadline)?; - let stdout = stdout_reader.collect(&deadline)?; - let stderr = stderr_reader.collect(&deadline)?; - refuse_exceeded(program, "stdout", &stdout)?; - refuse_exceeded(program, "stderr", &stderr)?; - Ok(ProcessOutput { - code: status.code(), - succeeded: status.success(), - stdout: stdout.bytes, - stderr: stderr.bytes, - }) -} - -fn wait_for_child( - program: &'static str, - child: &mut Child, - deadline: &ProcessDeadline, -) -> Result { - let ProcessDeadline::Bounded { duration, expires } = deadline else { - return match child.wait() { - Ok(status) => Ok(status), - Err(source) => Err(cleanup( - child, - ProcessError::Io { - program, - action: "wait", - source, - }, - )), - }; - }; - loop { - match child.try_wait() { - Err(source) => { - return Err(cleanup( - child, - ProcessError::Io { - program, - action: "poll", - source, - }, - )); - } - Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() >= *expires => { - return Err(cleanup( - child, - ProcessError::Timeout { - program, - duration: *duration, - }, - )); - } - Ok(None) => thread::sleep( - Duration::from_millis(10).min(expires.saturating_duration_since(Instant::now())), - ), - } - } -} - -const fn refuse_exceeded( - program: &'static str, - stream: &'static str, - output: &BoundedBytes, -) -> Result<(), ProcessError> { - if output.exceeded { - Err(ProcessError::OutputLimit { - program, - stream, - maximum: OUTPUT_LIMIT, - }) - } else { - Ok(()) - } -} - -fn cleanup(child: &mut std::process::Child, primary: ProcessError) -> ProcessError { - let kill = child.kill(); - let wait = child.wait(); - if let Err(source) = kill - && source.kind() != io::ErrorKind::InvalidInput - { - return ProcessError::Cleanup { - primary: Box::new(primary), - action: "kill", - source, - }; - } - if let Err(source) = wait { - return ProcessError::Cleanup { - primary: Box::new(primary), - action: "wait", - source, - }; - } - primary -} - #[cfg(test)] #[path = "bounded_process/tests.rs"] mod tests; diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs new file mode 100644 index 0000000..7554ce9 --- /dev/null +++ b/xtask/src/bounded_process/capture.rs @@ -0,0 +1,174 @@ +//! This module owns bounded child-process capture and collection. + +use std::os::unix::process::CommandExt; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::cleanup::{cleanup_process, join_after_cleanup, join_readers}; +use super::{ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker}; +use crate::process_output::BoundedBytes; + +const OUTPUT_LIMIT: usize = 1_048_576; + +pub(crate) fn capture( + program: &'static str, + command: &mut Command, + deadline: Option, +) -> Result { + let deadline = ProcessDeadline::new(program, deadline)?; + CapturedProcess::start(program, command)?.finish(program, &deadline) +} + +struct CapturedProcess { + child: Child, + stderr: ReaderWorker, + stdout: ReaderWorker, +} + +impl CapturedProcess { + fn start(program: &'static str, command: &mut Command) -> Result { + command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + let mut child = command.spawn().map_err(|source| ProcessError::Io { + program, + action: "spawn", + source, + })?; + let stdout = child.stdout.take().ok_or(ProcessError::MissingStream { + program, + stream: "stdout", + }); + let stderr = child.stderr.take().ok_or(ProcessError::MissingStream { + program, + stream: "stderr", + }); + let (stdout, stderr) = match (stdout, stderr) { + (Ok(stdout), Ok(stderr)) => (stdout, stderr), + (Err(error), _) | (_, Err(error)) => { + return Err(cleanup_process(&mut child, error)); + } + }; + let stdout = match ReaderWorker::start(program, "stdout", stdout, OUTPUT_LIMIT) { + Ok(reader) => reader, + Err(error) => return Err(cleanup_process(&mut child, error)), + }; + let stderr = match ReaderWorker::start(program, "stderr", stderr, OUTPUT_LIMIT) { + Ok(reader) => reader, + Err(error) => { + let error = cleanup_process(&mut child, error); + return Err(join_after_cleanup(stdout, error)); + } + }; + Ok(Self { + child, + stderr, + stdout, + }) + } + + fn finish( + mut self, + program: &'static str, + deadline: &ProcessDeadline, + ) -> Result { + let status = match wait_for_child(program, &mut self.child, deadline) { + Ok(status) => status, + Err(error) => return Err(join_readers(self.stdout, self.stderr, error)), + }; + let stdout = match self.stdout.receive(deadline) { + Ok(output) => output, + Err(error) => return Err(self.cleanup_readers(error)), + }; + let stderr = match self.stderr.receive(deadline) { + Ok(output) => output, + Err(error) => return Err(self.cleanup_readers(error)), + }; + if let Err(error) = self.stdout.join() { + let error = cleanup_process(&mut self.child, error); + return Err(join_after_cleanup(self.stderr, error)); + } + if let Err(error) = self.stderr.join() { + return Err(cleanup_process(&mut self.child, error)); + } + refuse_exceeded(program, "stdout", &stdout) + .and_then(|()| refuse_exceeded(program, "stderr", &stderr)) + .map_err(|error| cleanup_process(&mut self.child, error))?; + Ok(ProcessOutput { + code: status.code(), + succeeded: status.success(), + stdout: stdout.bytes, + stderr: stderr.bytes, + }) + } + + fn cleanup_readers(self, error: ProcessError) -> ProcessError { + let mut child = self.child; + let error = cleanup_process(&mut child, error); + join_readers(self.stdout, self.stderr, error) + } +} + +fn wait_for_child( + program: &'static str, + child: &mut Child, + deadline: &ProcessDeadline, +) -> Result { + let ProcessDeadline::Bounded { duration, expires } = deadline else { + return child.wait().map_err(|source| { + cleanup_process( + child, + ProcessError::Io { + program, + action: "wait", + source, + }, + ) + }); + }; + loop { + match child.try_wait() { + Err(source) => { + return Err(cleanup_process( + child, + ProcessError::Io { + program, + action: "poll", + source, + }, + )); + } + Ok(Some(status)) => return Ok(status), + Ok(None) if Instant::now() >= *expires => { + return Err(cleanup_process( + child, + ProcessError::Timeout { + program, + duration: *duration, + }, + )); + } + Ok(None) => thread::sleep( + Duration::from_millis(10).min(expires.saturating_duration_since(Instant::now())), + ), + } + } +} + +const fn refuse_exceeded( + program: &'static str, + stream: &'static str, + output: &BoundedBytes, +) -> Result<(), ProcessError> { + if output.exceeded { + Err(ProcessError::OutputLimit { + program, + stream, + maximum: OUTPUT_LIMIT, + }) + } else { + Ok(()) + } +} diff --git a/xtask/src/bounded_process/cleanup.rs b/xtask/src/bounded_process/cleanup.rs new file mode 100644 index 0000000..df05746 --- /dev/null +++ b/xtask/src/bounded_process/cleanup.rs @@ -0,0 +1,56 @@ +//! This module owns failed child-process and reader teardown. + +use std::io; +use std::process::Child; + +use super::process_group::ProcessGroup; +use super::{ProcessError, ReaderWorker}; + +pub(super) fn cleanup_process(child: &mut Child, primary: ProcessError) -> ProcessError { + let process_group = ProcessGroup::for_child(child).and_then(ProcessGroup::terminate); + let kill = child.kill(); + let wait = child.wait(); + if let Err(source) = process_group { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "terminate child process group", + source, + }; + } + if let Err(source) = kill + && source.kind() != io::ErrorKind::InvalidInput + { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "kill child process", + source, + }; + } + if let Err(source) = wait { + return ProcessError::Cleanup { + primary: Box::new(primary), + action: "reap child process", + source, + }; + } + primary +} + +pub(super) fn join_readers( + stdout: ReaderWorker, + stderr: ReaderWorker, + primary: ProcessError, +) -> ProcessError { + let primary = join_after_cleanup(stdout, primary); + join_after_cleanup(stderr, primary) +} + +pub(super) fn join_after_cleanup(reader: ReaderWorker, primary: ProcessError) -> ProcessError { + match reader.join() { + Ok(()) => primary, + Err(additional) => ProcessError::Additional { + primary: Box::new(primary), + additional: Box::new(additional), + }, + } +} diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs index d4e16eb..3ef6641 100644 --- a/xtask/src/bounded_process/error.rs +++ b/xtask/src/bounded_process/error.rs @@ -6,6 +6,10 @@ use std::io; use std::time::Duration; pub(crate) enum ProcessError { + Additional { + primary: Box, + additional: Box, + }, Cleanup { primary: Box, action: &'static str, @@ -38,7 +42,9 @@ pub(crate) enum ProcessError { impl ProcessError { pub(crate) fn is_not_found(&self) -> bool { match self { - Self::Cleanup { primary, .. } => primary.is_not_found(), + Self::Additional { primary, .. } | Self::Cleanup { primary, .. } => { + primary.is_not_found() + } Self::Io { source, .. } => source.kind() == io::ErrorKind::NotFound, Self::MissingStream { .. } | Self::OutputLimit { .. } @@ -57,12 +63,13 @@ impl fmt::Debug for ProcessError { impl fmt::Display for ProcessError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Additional { + primary, + additional, + } => write!(formatter, "{primary}; additionally {additional}"), Self::Cleanup { primary, action, .. - } => write!( - formatter, - "{primary}; additionally failed to {action} child process" - ), + } => write!(formatter, "{primary}; additionally failed to {action}"), Self::Io { program, action, .. } => write!(formatter, "cannot {action} {program} process"), @@ -92,6 +99,7 @@ impl fmt::Display for ProcessError { impl Error for ProcessError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + Self::Additional { primary, .. } => Some(primary), Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), Self::MissingStream { .. } | Self::OutputLimit { .. } diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs new file mode 100644 index 0000000..50100d1 --- /dev/null +++ b/xtask/src/bounded_process/process_group.rs @@ -0,0 +1,60 @@ +//! This module owns child process-group creation and termination. + +use std::io; +use std::process::Child; + +use rustix::io::Errno; +use rustix::process::{Pid, Signal, kill_process_group}; + +pub(super) struct ProcessGroup(Pid); + +impl ProcessGroup { + pub(super) fn for_child(child: &Child) -> Result { + let raw = i32::try_from(child.id()) + .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source))?; + let pid = Pid::from_raw(raw) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "child PID is zero"))?; + Ok(Self(pid)) + } + + pub(super) fn terminate(self) -> Result<(), io::Error> { + match kill_process_group(self.0, Signal::KILL) { + Ok(()) | Err(Errno::SRCH) => Ok(()), + Err(source) => Err(source.into()), + } + } +} + +#[cfg(test)] +const DESCENDANT_CHILD: &str = "KEEP_XTASK_DESCENDANT_CHILD"; +#[cfg(test)] +const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; +#[cfg(test)] +const DESCENDANT_READY: &str = "KEEP_XTASK_DESCENDANT_READY"; +#[cfg(test)] +const DESCENDANT_SOCKET: &str = "KEEP_XTASK_DESCENDANT_SOCKET"; + +#[cfg(test)] +fn wait_for_ready(path: &std::path::Path) -> Result<(), io::Error> { + let expires = std::time::Instant::now() + .checked_add(std::time::Duration::from_secs(2)) + .ok_or_else(|| io::Error::other("descendant readiness deadline overflow"))?; + while !path.is_file() { + if std::time::Instant::now() >= expires { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "descendant did not become ready", + )); + } + std::thread::yield_now(); + } + Ok(()) +} + +#[cfg(test)] +#[path = "process_group/child_tests.rs"] +mod child_tests; + +#[cfg(test)] +#[path = "process_group/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/process_group/child_tests.rs b/xtask/src/bounded_process/process_group/child_tests.rs new file mode 100644 index 0000000..2a234d0 --- /dev/null +++ b/xtask/src/bounded_process/process_group/child_tests.rs @@ -0,0 +1,70 @@ +//! This module owns subprocess fixtures for process-group regression tests. + +use std::env; +use std::fs; +use std::io; +use std::os::unix::net::UnixListener; +use std::path::Path; +use std::process::Command; + +use super::{ + DESCENDANT_CHILD, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, wait_for_ready, +}; + +#[test] +fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { + if env::var_os(DESCENDANT_PARENT).is_none() { + return Ok(()); + } + let executable = env::current_exe()?; + let ready = env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?; + drop( + Command::new(executable) + .args([ + "--exact", + "bounded_process::process_group::child_tests::process_descendant_holds_pipe", + ]) + .env(DESCENDANT_CHILD, "1") + .env(DESCENDANT_READY, &ready) + .env( + DESCENDANT_SOCKET, + env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant socket path", + ) + })?, + ) + .spawn()?, + ); + wait_for_ready(Path::new(&ready))?; + Ok(()) +} + +#[test] +fn process_descendant_holds_pipe() -> Result<(), io::Error> { + if env::var_os(DESCENDANT_CHILD).is_none() { + return Ok(()); + } + let socket = env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant socket path", + ) + })?; + let ready = env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?; + let listener = UnixListener::bind(socket)?; + fs::write(ready, b"ready")?; + loop { + let (mut stream, _) = listener.accept()?; + let mut command = [0_u8; 1]; + io::Read::read_exact(&mut stream, &mut command)?; + if command == [b'x'] { + return Ok(()); + } + } +} diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs new file mode 100644 index 0000000..67fab14 --- /dev/null +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -0,0 +1,122 @@ +//! This module owns child process-group cleanup regression evidence. + +use std::env; +use std::io::{self, Write}; +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use super::{DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, wait_for_ready}; +use crate::bounded_process::cleanup::cleanup_process; +use crate::bounded_process::{ProcessError, capture}; +use crate::test_directory::TestDirectory; + +const CHILD_PROCESS: &str = + "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open"; + +#[test] +fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> +{ + let directory = TestDirectory::create("process-descendant-deadline")?; + let ready = directory.path().join("ready"); + let socket = directory.path().join("descendant.sock"); + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args(["--exact", CHILD_PROCESS]) + .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_READY, &ready) + .env(DESCENDANT_SOCKET, &socket) + .stdin(Stdio::null()); + + let result = capture( + "test process", + &mut command, + Some(Duration::from_millis(25)), + ); + + assert!(matches!( + result, + Err(ProcessError::Timeout { + program: "test process", + duration, + }) if duration == Duration::from_millis(25) + )); + directory.close()?; + Ok(()) +} + +#[test] +fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box> { + let directory = TestDirectory::create("process-group-cleanup")?; + let ready = directory.path().join("ready"); + let socket = directory.path().join("descendant.sock"); + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args(["--exact", CHILD_PROCESS]) + .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_READY, &ready) + .env(DESCENDANT_SOCKET, &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .process_group(0); + let mut child = command.spawn()?; + wait_for_ready(&ready)?; + + let error = cleanup_process( + &mut child, + ProcessError::Timeout { + program: "test process", + duration: Duration::from_millis(25), + }, + ); + + assert!(matches!(error, ProcessError::Timeout { .. })); + let descendant_survived = descendant_survived_cleanup(&socket)?; + directory.close()?; + assert!( + !descendant_survived, + "cleanup returned while a descendant remained reachable" + ); + Ok(()) +} + +fn descendant_survived_cleanup(socket: &Path) -> Result { + let expires = Instant::now() + .checked_add(Duration::from_millis(500)) + .ok_or_else(|| io::Error::other("descendant cleanup deadline overflow"))?; + loop { + match UnixStream::connect(socket) { + Ok(mut stream) if Instant::now() >= expires => { + stream.write_all(b"x")?; + return Ok(true); + } + Ok(mut stream) => match stream.write_all(b"p") { + Ok(()) => thread::yield_now(), + Err(source) + if matches!( + source.kind(), + io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ) => + { + return Ok(false); + } + Err(source) => return Err(source), + }, + Err(source) + if matches!( + source.kind(), + io::ErrorKind::ConnectionRefused | io::ErrorKind::NotFound + ) => + { + return Ok(false); + } + Err(source) => return Err(source), + } + } +} diff --git a/xtask/src/bounded_process/reader.rs b/xtask/src/bounded_process/reader.rs index f373411..cecf26c 100644 --- a/xtask/src/bounded_process/reader.rs +++ b/xtask/src/bounded_process/reader.rs @@ -41,28 +41,29 @@ impl ReaderWorker { }) } - pub(super) fn collect(self, deadline: &ProcessDeadline) -> Result { - let Self { - handle, - program, - receiver, - stream, - } = self; - let result = match deadline.remaining(program)? { - Some((remaining, duration)) => receiver + pub(super) fn receive(&self, deadline: &ProcessDeadline) -> Result { + let result = match deadline.remaining(self.program)? { + Some((remaining, duration)) => self + .receiver .recv_timeout(remaining) - .map_err(|error| receive_error(program, stream, duration, error))?, - None => receiver.recv().map_err(|_| reader_panic(program, stream))?, + .map_err(|error| receive_error(self.program, self.stream, duration, error))?, + None => self + .receiver + .recv() + .map_err(|_| reader_panic(self.program, self.stream))?, }; - handle - .join() - .map_err(|_panic| reader_panic(program, stream))?; result.map_err(|source| ProcessError::Io { - program, + program: self.program, action: "read child output", source, }) } + + pub(super) fn join(self) -> Result<(), ProcessError> { + self.handle + .join() + .map_err(|_panic| reader_panic(self.program, self.stream)) + } } const fn receive_error( diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index ddae442..4a419ae 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -3,13 +3,10 @@ use std::env; use std::io::{self, Write}; use std::process::{Command, Stdio}; -use std::sync::mpsc::{self, RecvTimeoutError}; use std::time::Duration; use super::{ProcessError, capture}; -const DESCENDANT_CHILD: &str = "KEEP_XTASK_DESCENDANT_CHILD"; -const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; const OUTPUT_CHILD: &str = "KEEP_XTASK_BOUNDED_OUTPUT_CHILD"; #[test] @@ -48,63 +45,3 @@ fn process_child_writes_excess_output() -> Result<(), io::Error> { output.write_all(&bytes)?; output.flush() } - -#[test] -fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> -{ - let executable = env::current_exe()?; - let mut command = Command::new(executable); - command - .args([ - "--exact", - "bounded_process::tests::process_child_leaves_descendant_pipe_open", - ]) - .env(DESCENDANT_PARENT, "1") - .stdin(Stdio::null()); - - let result = capture( - "test process", - &mut command, - Some(Duration::from_millis(25)), - ); - - assert!(matches!( - result, - Err(ProcessError::Timeout { - program: "test process", - duration, - }) if duration == Duration::from_millis(25) - )); - Ok(()) -} - -#[test] -fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { - if env::var_os(DESCENDANT_PARENT).is_none() { - return Ok(()); - } - let executable = env::current_exe()?; - drop( - Command::new(executable) - .args([ - "--exact", - "bounded_process::tests::process_descendant_holds_pipe", - ]) - .env(DESCENDANT_CHILD, "1") - .spawn()?, - ); - Ok(()) -} - -#[test] -fn process_descendant_holds_pipe() { - if env::var_os(DESCENDANT_CHILD).is_none() { - return; - } - let (sender, receiver) = mpsc::channel::<()>(); - assert!(matches!( - receiver.recv_timeout(Duration::from_millis(250)), - Err(RecvTimeoutError::Timeout) - )); - drop(sender); -} From ba4fd74b7e2e38b0ed26e826507865a228a7ba69 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:45:32 -0700 Subject: [PATCH 022/113] Fix: refuse disguised Python sources --- CHANGELOG.md | 3 +- docs/Rust Standards.md | 2 + xtask/src/source_structure.rs | 10 +- xtask/src/source_structure/pure_rust_tests.rs | 63 ++++++++- xtask/src/source_structure/python_source.rs | 131 ++++++++++++++++++ xtask/src/source_structure/source_kind.rs | 23 ++- xtask/src/source_structure/tests.rs | 6 +- 7 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 xtask/src/source_structure/python_source.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c5434f..87c0498 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,8 @@ after its public API and format compatibility policies are established. - Repository source verification now uses capability-relative, no-follow file opens and verifies repository-root identity after Git inventory, so a persistent root replacement or source path replaced with a symlink is - refused before source bytes are read. + refused before source bytes are read. The pure Rust boundary also refuses + `.py`, `.pyw`, and extensionless executable Python shebangs. - The repository `cargo xtask` alias and Rust command contract are now explicitly silent on success and emit one typed `Error:` diagnostic with exit status 1 on refusal; untrusted control characters are escaped so the diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 6f6775e..33da1ea 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -2041,6 +2041,8 @@ Also forbidden: - hidden filesystem access; - hidden network access; - hidden allocation proportional to input; +- Python source files, including `.py`, `.pyw`, and extensionless executable + Python shebangs; - random IDs where content identity is required; - wall-clock time in deterministic algorithms; - hashing arbitrary serializer output; diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 208ea25..60eb4f5 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -1,5 +1,6 @@ //! This module owns source inventory orchestration and the 500-line law. +mod python_source; mod repository_path; mod source_error; mod source_kind; @@ -10,9 +11,10 @@ use std::path::Path; use crate::git_inventory::{GitPath, paths as git_paths}; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +use python_source::refuse_executable_python; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; -use source_kind::{is_python_module, is_source_module}; +use source_kind::{is_extensionless_file, is_python_module, is_source_candidate}; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ @@ -78,7 +80,7 @@ fn select_source_paths( ) -> Result, SourceStructureError> { present .difference(deleted) - .filter(|path| is_source_module(path.as_bytes())) + .filter(|path| is_source_candidate(path.as_bytes())) .map(admit_source_path) .collect() } @@ -107,6 +109,10 @@ fn source_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { + if is_extensionless_file(relative.as_str().as_bytes()) { + refuse_executable_python(source_root, &relative)?; + continue; + } let lines = source_line_count(source_root, &relative)?; if lines == SourceLineCount::Exceeded { violations.push(relative.as_str().to_owned()); diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 538e038..77ec43b 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -1,10 +1,21 @@ //! This module owns the pure-Rust source-admission regression law. +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::fs::PermissionsExt; + use crate::git_inventory::GitPath; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; #[test] fn python_source_is_refused_by_the_pure_rust_boundary() { - for path in ["scripts/check.py", "scripts/check.PY"] { + for path in [ + "scripts/check.py", + "scripts/check.PY", + "scripts/check.pyw", + "scripts/check.PYW", + ] { assert!(matches!( super::admit_source_path(&GitPath::new(path.as_bytes().to_vec())), Err(super::SourceStructureError::PythonSource(ref observed)) @@ -12,3 +23,53 @@ fn python_source_is_refused_by_the_pure_rust_boundary() { )); } } + +#[test] +fn extensionless_executable_python_is_refused_by_the_pure_rust_boundary() +-> Result<(), Box> { + let directory = TestDirectory::create("extensionless-python")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check"); + fs::write(&script, b"#!/usr/bin/env python3\nprint('forbidden')\n")?; + let mut permissions = fs::metadata(&script)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&script, permissions)?; + let present = BTreeSet::from([GitPath::new(b"check".to_vec())]); + + let paths = super::select_source_paths(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let result = super::source_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) if path == "check" + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn extensionless_nonexecutable_text_is_not_a_source_module() +-> Result<(), Box> { + let directory = TestDirectory::create("extensionless-text")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + fs::write( + repository.join("NOTICE"), + b"#!/usr/bin/env python3\nnot executable\n", + )?; + let mut permissions = fs::metadata(repository.join("NOTICE"))?.permissions(); + permissions.set_mode(0o644); + fs::set_permissions(repository.join("NOTICE"), permissions)?; + let present = BTreeSet::from([GitPath::new(b"NOTICE".to_vec())]); + + let paths = super::select_source_paths(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + assert!(super::source_violations(&source_root, paths)?.is_empty()); + + drop(source_root); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs new file mode 100644 index 0000000..ee23236 --- /dev/null +++ b/xtask/src/source_structure/python_source.rs @@ -0,0 +1,131 @@ +//! This module owns bounded executable Python-shebang admission. + +use std::fs::File; +use std::io::{self, Read}; +use std::os::unix::fs::PermissionsExt; + +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; + +use super::SourceStructureError; +use super::repository_path::RepositoryPath; + +const SHEBANG_SCAN_BYTES: u64 = 1_024; + +pub(super) fn refuse_executable_python( + source_root: &RepositoryRoot, + relative: &RepositoryPath, +) -> Result<(), SourceStructureError> { + let path = source_root.display_path(relative.as_path()); + let file = source_root + .open_file(relative.as_path()) + .map_err(|error| match error { + OpenRepositoryFileError::Io(source) => SourceStructureError::Inspect { + path: path.clone(), + source, + }, + OpenRepositoryFileError::NonRegular => SourceStructureError::NonRegular(path.clone()), + })?; + let python = executable_uses_python(&file).map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; + if python { + Err(SourceStructureError::PythonSource( + relative.as_str().to_owned(), + )) + } else { + Ok(()) + } +} + +fn executable_uses_python(file: &File) -> Result { + if file.metadata()?.permissions().mode() & 0o111 == 0 { + return Ok(false); + } + let mut prefix = Vec::new(); + file.take(SHEBANG_SCAN_BYTES).read_to_end(&mut prefix)?; + Ok(is_python_shebang(&prefix)) +} + +fn is_python_shebang(prefix: &[u8]) -> bool { + let Some(line) = prefix + .split(|byte| *byte == b'\n') + .next() + .and_then(|line| line.strip_prefix(b"#!")) + else { + return false; + }; + let mut words = line + .split(u8::is_ascii_whitespace) + .filter(|word| !word.is_empty()); + let Some(interpreter) = words.next() else { + return false; + }; + if is_python_program(interpreter) { + return true; + } + if !program_name(interpreter).eq_ignore_ascii_case(b"env") { + return false; + } + words.any(environment_word_selects_python) +} + +fn environment_word_selects_python(word: &[u8]) -> bool { + if let Some(split) = word.strip_prefix(b"--split-string=") { + return is_python_program(split); + } + !word.starts_with(b"-") && !word.contains(&b'=') && is_python_program(word) +} + +fn is_python_program(program: &[u8]) -> bool { + let unquoted = program + .strip_prefix(b"\"") + .or_else(|| program.strip_prefix(b"'")) + .unwrap_or(program); + let name = program_name(unquoted); + starts_with_ignore_ascii_case(name, b"python") || starts_with_ignore_ascii_case(name, b"pypy") +} + +fn program_name(program: &[u8]) -> &[u8] { + program + .rsplit(|byte| *byte == b'/') + .next() + .unwrap_or(program) +} + +fn starts_with_ignore_ascii_case(value: &[u8], prefix: &[u8]) -> bool { + value + .get(..prefix.len()) + .is_some_and(|observed| observed.eq_ignore_ascii_case(prefix)) +} + +#[cfg(test)] +mod tests { + use super::is_python_shebang; + + #[test] + fn direct_and_environment_python_interpreters_are_detected() { + for shebang in [ + b"#!/usr/bin/python3\n".as_slice(), + b"#! /usr/bin/env python3 -I\n", + b"#!/usr/bin/env -S python3 -I\n", + b"#!/usr/bin/env -S \"python3 -I\"\n", + b"#!/usr/bin/env --split-string=python3\n", + b"#!/opt/PyPy3\n", + ] { + assert!(is_python_shebang(shebang)); + } + } + + #[test] + fn non_python_or_displaced_interpreters_are_not_misclassified() { + for prefix in [ + b"#!/bin/sh\n".as_slice(), + b"#!/usr/bin/env bash\n", + b"python3\n", + b"first line\n#!/usr/bin/python3\n", + ] { + assert!(!is_python_shebang(prefix)); + } + } +} diff --git a/xtask/src/source_structure/source_kind.rs b/xtask/src/source_structure/source_kind.rs index 021a1bc..ee97792 100644 --- a/xtask/src/source_structure/source_kind.rs +++ b/xtask/src/source_structure/source_kind.rs @@ -1,18 +1,29 @@ //! This module owns repository source-module classification. -const SOURCE_SUFFIXES: [[u8; 2]; 3] = [*b"py", *b"rs", *b"sh"]; - pub(super) fn is_source_module(path: &[u8]) -> bool { let Some(suffix) = source_suffix(path) else { return false; }; - SOURCE_SUFFIXES.iter().any(|candidate| { - suffix == candidate || (*candidate == *b"py" && suffix.eq_ignore_ascii_case(b"py")) - }) + suffix == b"rs" || suffix == b"sh" || is_python_suffix(suffix) } pub(super) fn is_python_module(path: &[u8]) -> bool { - source_suffix(path).is_some_and(|suffix| suffix.eq_ignore_ascii_case(b"py")) + source_suffix(path).is_some_and(is_python_suffix) +} + +pub(super) fn is_source_candidate(path: &[u8]) -> bool { + is_source_module(path) || is_extensionless_file(path) +} + +pub(super) fn is_extensionless_file(path: &[u8]) -> bool { + let Some(file_name) = path.rsplit(|byte| *byte == b'/').next() else { + return false; + }; + !file_name.is_empty() && !file_name.contains(&b'.') +} + +const fn is_python_suffix(suffix: &[u8]) -> bool { + suffix.eq_ignore_ascii_case(b"py") || suffix.eq_ignore_ascii_case(b"pyw") } fn source_suffix(path: &[u8]) -> Option<&[u8]> { diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index 01e8721..44f79f0 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -1,8 +1,7 @@ //! This module owns source-line, selection, and replacement-race tests. -use super::{ - PRESENT_PATH_ARGUMENTS, SourceLineCount, exceeds_hard_limit, is_source_module, line_count, -}; +use super::source_kind::is_source_module; +use super::{PRESENT_PATH_ARGUMENTS, SourceLineCount, exceeds_hard_limit, line_count}; use crate::test_directory::TestDirectory; use std::io::{self, BufReader, Cursor, Read}; @@ -276,6 +275,7 @@ fn source_module_limit_accepts_five_hundred_and_refuses_five_hundred_one() { fn source_module_classification_is_explicit() { assert!(is_source_module(b"src/lib.rs")); assert!(is_source_module(b"scripts/check.py")); + assert!(is_source_module(b"scripts/check.pyw")); assert!(is_source_module(b"scripts/check.sh")); assert!(!is_source_module(b"README.md")); assert!(!is_source_module(b"src/lib.RS")); From 653bb984fc48390133e57618372883f1e021e3ac Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 13:59:00 -0700 Subject: [PATCH 023/113] Fix: bind the reviewed Node lock graph --- CHANGELOG.md | 5 +- docs/dependencies/documentation-toolchain.md | 8 ++- .../documentation_integrity/node_toolchain.rs | 15 ++++- .../node_toolchain/tests.rs | 3 + .../node_toolchain/tests/lock_graph.rs | 63 +++++++++++++++++++ 5 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c0498..62d7ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,9 @@ after its public API and format compatibility policies are established. run through bounded Rust `xtask` code; CI and `cargo xtask verify` use that boundary, and the seven superseded Python checkers have been removed. The boundary rejects duplicate repository JSON fields and unlocked installer - substitutions, retains simultaneous Markdown and link failures, and applies - one deadline across child execution and output collection. + substitutions, admits only the exact reviewed Node lock artifact, retains + simultaneous Markdown and link failures, and applies one deadline across + child execution and output collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index f04803b..a6316da 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -32,9 +32,11 @@ before extraction: | `lychee-x86_64-unknown-linux-gnu.tar.gz` | `a06547250f10021dcafc6ed5bb20fca75835b65711745b63cfdda34c29ff6a73` | The Rust `cargo xtask documentation-integrity-check` boundary verifies the -committed Node graph and the exact BLAKE3 digest of the reviewed installer, -then verifies each executable's reported version before admitting its output -as evidence. A missing tool, changed archive, unexpected version, empty input +structure and exact BLAKE3 digest of the committed Node lock artifact and the +exact BLAKE3 digest of the reviewed installer. The byte-exact lock admission +refuses altered, omitted, or additional package records. The boundary then +verifies each executable's reported version before admitting its output as +evidence. A missing tool, changed archive, unexpected version, empty input corpus, or tool failure refuses the check. ## Determinism and network posture diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index bb99480..3ac55d5 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -14,6 +14,10 @@ const INSTALLER_DIGEST: [u8; 32] = [ 0x12, 0xfb, 0x82, 0xcd, 0xdb, 0x65, 0x52, 0xe5, 0xae, 0xdb, 0x14, 0x54, 0x83, 0xf4, 0x8a, 0x8a, 0x8b, 0x35, 0x54, 0xea, 0x2a, 0x90, 0xeb, 0xd4, 0x82, 0xd7, 0x4a, 0x61, 0x1f, 0xf6, 0xd3, 0xfd, ]; +const LOCK_DIGEST: [u8; 32] = [ + 0x74, 0x21, 0xce, 0x90, 0xdd, 0x52, 0x33, 0xfe, 0x99, 0x1a, 0x0b, 0x7e, 0xdd, 0xaa, 0xb7, 0x53, + 0x63, 0xf3, 0xad, 0x3b, 0x0f, 0x9e, 0x7d, 0xa2, 0xa4, 0x77, 0x65, 0xf0, 0x9c, 0x1d, 0xcb, 0x3b, +]; const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; @@ -21,7 +25,8 @@ pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), Documentatio let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; let lock = repository_text::read(repository_root, LOCK_PATH)?; let installer = repository_text::read(repository_root, INSTALLER_PATH)?; - admit(&manifest, &lock, &installer) + admit(&manifest, &lock, &installer)?; + admit_lock_bytes(&lock) } fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), DocumentationError> { @@ -105,6 +110,14 @@ fn admit_installer(installer: &str) -> Result<(), DocumentationError> { } } +fn admit_lock_bytes(lock: &str) -> Result<(), DocumentationError> { + if blake3::hash(lock.as_bytes()).as_bytes() == &LOCK_DIGEST { + Ok(()) + } else { + Err(contract(LOCK_PATH, "lock bytes match the reviewed digest")) + } +} + fn parse(path: &'static str, raw: &str) -> Result { unique_json::parse(raw).map_err(|source| DocumentationError::RepositoryJson { path, source }) } diff --git a/xtask/src/documentation_integrity/node_toolchain/tests.rs b/xtask/src/documentation_integrity/node_toolchain/tests.rs index e672f05..1336533 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests.rs @@ -2,6 +2,9 @@ use std::path::Path; use crate::repository_file::RepositoryRoot; +#[path = "tests/lock_graph.rs"] +mod lock_graph; + const MANIFEST: &str = r#"{"dependencies":{"markdownlint-cli2":"0.23.2"}}"#; const LOCK: &str = r#"{ "lockfileVersion": 3, diff --git a/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs new file mode 100644 index 0000000..fd41e4b --- /dev/null +++ b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs @@ -0,0 +1,63 @@ +use std::fs; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::super::{DocumentationError, INSTALLER_PATH, LOCK_PATH, MANIFEST_PATH}; + +const INSTALLER: &str = include_str!("../../../../../scripts/install_documentation_tools.sh"); +const LOCK: &str = include_str!("../../../../../scripts/documentation-tools/package-lock.json"); +const MANIFEST: &str = include_str!("../../../../../scripts/documentation-tools/package.json"); + +#[test] +fn unreviewed_lock_graph_changes_are_refused() -> Result<(), Box> { + let altered = LOCK.replacen( + "\"resolved\": \"https://registry.npmjs.org/", + "\"resolved\": \"https://example.com/", + 1, + ); + let extra = LOCK.replacen( + "\n }\n}", + concat!( + ",\n", + " \"node_modules/unreviewed\": {\n", + " \"version\": \"1.0.0\",\n", + " \"resolved\": \"https://example.com/unreviewed.tgz\",\n", + " \"integrity\": \"sha512-example\"\n", + " }\n", + " }\n", + "}" + ), + 1, + ); + + for lock in [altered, extra] { + assert!(matches!( + check_with_lock(&lock)?, + Err(DocumentationError::RepositoryContract { + path: LOCK_PATH, + requirement: "lock bytes match the reviewed digest", + }) + )); + } + Ok(()) +} + +fn check_with_lock( + lock: &str, +) -> Result, Box> { + let repository = TestDirectory::create("node-lock-graph")?; + write_repository(&repository, lock)?; + let root = RepositoryRoot::open(repository.path())?; + let result = super::super::check(&root); + repository.close()?; + Ok(result) +} + +fn write_repository(repository: &TestDirectory, lock: &str) -> Result<(), std::io::Error> { + let tool_directory = repository.path().join("scripts/documentation-tools"); + fs::create_dir_all(&tool_directory)?; + fs::write(repository.path().join(MANIFEST_PATH), MANIFEST)?; + fs::write(repository.path().join(LOCK_PATH), lock)?; + fs::write(repository.path().join(INSTALLER_PATH), INSTALLER) +} From 3367b12cb425b66440b50b7e4ef4733ef4e2b878 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:05:18 -0700 Subject: [PATCH 024/113] Fix: parse documentation workflow commands --- CHANGELOG.md | 5 +- Cargo.lock | 41 +++++++++ docs/dependencies/documentation-toolchain.md | 10 ++- docs/dependencies/yaml-rust2-0.11.0.md | 68 +++++++++++++++ xtask/Cargo.toml | 3 + xtask/src/documentation_integrity/error.rs | 5 ++ .../documentation_integrity/error/display.rs | 3 + .../workflow_contract.rs | 83 ++++++++++++------- .../workflow_contract/tests.rs | 45 +++++++++- 9 files changed, 226 insertions(+), 37 deletions(-) create mode 100644 docs/dependencies/yaml-rust2-0.11.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d7ae1..90f6da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,9 @@ after its public API and format compatibility policies are established. boundary, and the seven superseded Python checkers have been removed. The boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains - simultaneous Markdown and link failures, and applies one deadline across - child execution and output collection. + simultaneous Markdown and link failures, parses documentation workflow + commands as YAML, and applies one deadline across child execution and output + collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/Cargo.lock b/Cargo.lock index 4e671a7..b0ac41a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.9" @@ -235,6 +241,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "fs-set-times" version = "0.20.3" @@ -246,6 +258,24 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown", +] + [[package]] name = "hybrid-array" version = "0.4.13" @@ -681,6 +711,17 @@ dependencies = [ "rustix", "serde", "serde_json", + "yaml-rust2", +] + +[[package]] +name = "yaml-rust2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" +dependencies = [ + "arraydeque", + "hashlink", ] [[package]] diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index a6316da..b2248e4 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -34,10 +34,12 @@ before extraction: The Rust `cargo xtask documentation-integrity-check` boundary verifies the structure and exact BLAKE3 digest of the committed Node lock artifact and the exact BLAKE3 digest of the reviewed installer. The byte-exact lock admission -refuses altered, omitted, or additional package records. The boundary then -verifies each executable's reported version before admitting its output as -evidence. A missing tool, changed archive, unexpected version, empty input -corpus, or tool failure refuses the check. +refuses altered, omitted, or additional package records. The boundary parses +the CI workflow as YAML and admits only reviewed `run` fields from the +documentation job, then verifies each executable's reported version before +admitting its output as evidence. A missing tool, changed archive, unexpected +version, empty input corpus, unreviewed command, or tool failure refuses the +check. ## Determinism and network posture diff --git a/docs/dependencies/yaml-rust2-0.11.0.md b/docs/dependencies/yaml-rust2-0.11.0.md new file mode 100644 index 0000000..e189066 --- /dev/null +++ b/docs/dependencies/yaml-rust2-0.11.0.md @@ -0,0 +1,68 @@ +# Dependency Admission: yaml-rust2 0.11.0 + +- Status: Accepted for repository-task workflow admission only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [Ethiraric/yaml-rust2](https://github.com/Ethiraric/yaml-rust2) + +## Admitted use + +Keep admits exactly pinned `yaml-rust2` 0.11.0 only behind the `xtask` crate's +`repository-tasks` feature. The documentation-integrity task parses +`.github/workflows/ci.yml`, selects the `documentation` job's actual `run` +fields, and admits only the reviewed command set. Comments, display strings, +unrelated fields, and additional shell commands cannot satisfy that execution +contract. + +The dependency is absent from Keep's published library graph, public API, +content identities, durable formats, and production behavior. Its typed parse +error remains inside the private repository-task adapter. + +## Why a dependency is needed + +YAML includes quoted and block scalars, comments, aliases, nested collections, +and duplicate mapping keys. A substring scan cannot distinguish executable +`run` fields from inert text. A maintained parser keeps the workflow boundary +structural and fail-closed without creating a partial YAML implementation +inside Keep. + +The parsed values are never hashed, persisted, or admitted as Keep domain +types. The task reads the fixed workflow path through the bounded, +capability-relative, no-follow repository-file boundary before parsing it. + +## Features and resolved graph + +The direct dependency disables default features and is optional. It is +activated solely by `repository-tasks`; disabling defaults excludes the +optional non-UTF-8 input support. + +The introduced normal dependency graph is: + +- `arraydeque` 0.5.1; +- `foldhash` 0.2.0; +- `hashbrown` 0.16.1; and +- `hashlink` 0.11.1. + +## Safety, licensing, and compatibility + +`yaml-rust2` declares the MIT OR Apache-2.0 license expression and a minimum +supported Rust version of 1.65, below Keep's pinned toolchain. Its 0.11.0 Rust +source contains no `unsafe` block. Keep-owned code invokes only safe APIs. + +`cargo deny check licenses bans sources` and `cargo audit` pass with the +resolved graph. These checks remain mandatory point-in-time evidence. + +## Failure and recovery boundaries + +Malformed YAML, duplicate mapping keys, an absent documentation job, an +unreviewed command, an oversized workflow, a non-UTF-8 workflow, or a replaced +repository root produces a typed refusal. The task never repairs, rewrites, or +substitutes workflow data. Parsing has no durability or recovery semantics. + +Keep can remove this dependency without changing public or durable behavior by +replacing it with an equally bounded parser that preserves structural `run` +selection, duplicate-key refusal, and the reviewed-command laws. + +Reopen this admission if the direct version, selected features, resolved graph, +license, MSRV, repository-task-only boundary, or admitted YAML use changes. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 36f7475..4d437ab 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -17,6 +17,7 @@ repository-tasks = [ "dep:rustix", "dep:serde", "dep:serde_json", + "dep:yaml-rust2", ] [dependencies] @@ -33,6 +34,8 @@ rustix = { version = "=1.1.4", default-features = false, features = ["process", serde = { version = "=1.0.229", default-features = false, features = ["std"], optional = true } # Typed JSON admission checks the committed documentation-tool lock graph. serde_json = { version = "=1.0.151", default-features = false, features = ["std"], optional = true } +# Pure Rust YAML admission identifies executable GitHub Actions steps. +yaml-rust2 = { version = "=0.11.0", default-features = false, optional = true } [[bin]] name = "xtask" diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index e20a3b9..6542a00 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -62,6 +62,10 @@ pub(crate) enum DocumentationError { path: &'static str, source: serde_json::Error, }, + RepositoryYaml { + path: &'static str, + source: yaml_rust2::ScanError, + }, RepositoryRootChanged(PathBuf), RepositoryRootInspect { path: PathBuf, @@ -115,6 +119,7 @@ impl Error for DocumentationError { Some(source) } Self::RepositoryJson { source, .. } => Some(source), + Self::RepositoryYaml { source, .. } => Some(source), Self::RepositoryRootInspect { source, .. } => Some(source), Self::ToolOutputEncoding { source, .. } => Some(source), Self::ToolUnavailable { source, .. } => Some(source), diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 99dabf9..95584de 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -67,6 +67,9 @@ impl fmt::Display for DocumentationError { Self::RepositoryJson { path, .. } => { write!(formatter, "repository file `{path}` is not valid JSON") } + Self::RepositoryYaml { path, .. } => { + write!(formatter, "repository file `{path}` is not valid YAML") + } Self::RepositoryRootChanged(path) => { repository_root(formatter, RepositoryRootDiagnostic::Changed, path) } diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 234fe77..7f6603a 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -1,13 +1,26 @@ //! This module owns the CI documentation-job execution contract. +use yaml_rust2::YamlLoader; + use crate::repository_file::RepositoryRoot; use super::error::DocumentationError; use super::repository_text; const CI_PATH: &str = ".github/workflows/ci.yml"; -const DOCUMENTATION_JOB: &str = " documentation:"; -const XTASK_COMMAND: &str = "run: cargo xtask documentation-integrity-check"; +const XTASK_COMMAND: &str = "cargo xtask documentation-integrity-check"; +const REVIEWED_RUNS: &[&str] = &[ + "rustup show", + r#"documentation_tools="$RUNNER_TEMP/documentation-tools" +scripts/install_documentation_tools.sh "$documentation_tools" +printf '%s\n' \ + "$documentation_tools/bin" \ + "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH""#, + r"cargo test --locked --package xtask \ + documentation_integrity::execution::external_tests -- --ignored", + XTASK_COMMAND, + r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#, +]; pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { let workflow = repository_text::read(repository_root, CI_PATH)?; @@ -15,38 +28,48 @@ pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), Documentatio } fn admit(workflow: &str) -> Result<(), DocumentationError> { - let job = documentation_job(workflow)?; - if !job.contains("run: rustup show") { - return Err(contract( - "documentation job installs the pinned Rust toolchain", - )); - } - if job.matches(XTASK_COMMAND).count() != 1 { - return Err(contract( - "documentation job runs the Rust integrity command exactly once", - )); - } - if job.contains("python3") { - return Err(contract("documentation job contains no Python execution")); + let runs = documentation_runs(workflow)?; + if !runs_are_reviewed(&runs) { + return Err(contract(concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ))); } Ok(()) } -fn documentation_job(workflow: &str) -> Result { - let mut lines = workflow - .lines() - .skip_while(|line| *line != DOCUMENTATION_JOB); - if lines.next().is_none() { - return Err(contract("workflow defines the documentation job")); - } - let job: Vec<_> = lines - .take_while(|line| line.starts_with(" ") || line.is_empty() || !line.starts_with(" ")) - .collect(); - if job.is_empty() { - Err(contract("documentation job is not empty")) - } else { - Ok(job.join("\n")) - } +fn documentation_runs(workflow: &str) -> Result, DocumentationError> { + let documents = YamlLoader::load_from_str(workflow).map_err(|source| { + DocumentationError::RepositoryYaml { + path: CI_PATH, + source, + } + })?; + let [document] = documents.as_slice() else { + return Err(contract("workflow contains exactly one YAML document")); + }; + let Some(steps) = document["jobs"]["documentation"]["steps"].as_vec() else { + return Err(contract("workflow defines documentation job steps")); + }; + Ok(steps + .iter() + .filter_map(|step| step["run"].as_str()) + .map(|run| run.trim_end_matches('\n').to_owned()) + .collect()) +} + +fn runs_are_reviewed(runs: &[String]) -> bool { + runs.iter().all(|run| REVIEWED_RUNS.contains(&run.as_str())) + && runs + .iter() + .filter(|run| run.as_str() == "rustup show") + .count() + == 1 + && runs + .iter() + .filter(|run| run.as_str() == XTASK_COMMAND) + .count() + == 1 } const fn contract(requirement: &'static str) -> DocumentationError { diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index ff84708..f2e4517 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -30,7 +30,50 @@ fn documentation_job_refuses_python_execution() { super::admit(&workflow), Err(super::DocumentationError::RepositoryContract { path: super::CI_PATH, - requirement: "documentation job contains no Python execution", + requirement: concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ), + }) + )); +} + +#[test] +fn inert_yaml_cannot_impersonate_documentation_commands() { + let workflow = r#"name: CI +jobs: + documentation: + steps: + # run: rustup show + - name: "run: cargo xtask documentation-integrity-check" + uses: example/action@0123456789abcdef +"#; + assert!(matches!( + super::admit(workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ), + }) + )); +} + +#[test] +fn unreviewed_python_executables_are_refused() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Unreviewed executable\n run: python --version\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ), }) )); } From 7f691bd9ba0705cbbdced396748e6af688af05aa Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:07:12 -0700 Subject: [PATCH 025/113] Fix: enforce inherited fuzz deadlines --- CHANGELOG.md | 4 +-- xtask/src/bounded_process.rs | 11 ++++++-- xtask/src/bounded_process/capture.rs | 2 +- xtask/src/bounded_process/tests.rs | 41 +++++++++++++++++++++++++++- xtask/src/fuzz_campaign/execution.rs | 4 ++- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f6da8..0106a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,8 @@ after its public API and format compatibility policies are established. boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow - commands as YAML, and applies one deadline across child execution and output - collection. + commands as YAML, and applies one deadline across captured and inherited + child execution and output collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 4a73e37..0444df9 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -7,9 +7,12 @@ mod error; mod process_group; mod reader; +use std::os::unix::process::CommandExt; use std::process::Command; +use std::time::Duration; pub(crate) use capture::capture; +use capture::wait_for_child; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use reader::ReaderWorker; @@ -24,12 +27,16 @@ pub(crate) struct ProcessOutput { pub(crate) fn status( program: &'static str, command: &mut Command, + deadline: Option, ) -> Result { - let status = command.status().map_err(|source| ProcessError::Io { + let deadline = ProcessDeadline::new(program, deadline)?; + command.process_group(0); + let mut child = command.spawn().map_err(|source| ProcessError::Io { program, - action: "wait for", + action: "spawn", source, })?; + let status = wait_for_child(program, &mut child, &deadline)?; Ok(ProcessOutput { code: status.code(), succeeded: status.success(), diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 7554ce9..3dae7bf 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -111,7 +111,7 @@ impl CapturedProcess { } } -fn wait_for_child( +pub(super) fn wait_for_child( program: &'static str, child: &mut Child, deadline: &ProcessDeadline, diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index 4a419ae..4caddb5 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -5,9 +5,10 @@ use std::io::{self, Write}; use std::process::{Command, Stdio}; use std::time::Duration; -use super::{ProcessError, capture}; +use super::{ProcessError, capture, status}; const OUTPUT_CHILD: &str = "KEEP_XTASK_BOUNDED_OUTPUT_CHILD"; +const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_CHILD"; #[test] fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box> @@ -45,3 +46,41 @@ fn process_child_writes_excess_output() -> Result<(), io::Error> { output.write_all(&bytes)?; output.flush() } + +#[test] +fn inherited_process_obeys_the_process_deadline() -> Result<(), Box> { + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::tests::process_child_parks_indefinitely", + ]) + .env(PARKED_CHILD, "1") + .stdin(Stdio::null()); + + let result = status( + "test process", + &mut command, + Some(Duration::from_millis(50)), + ); + + assert!(matches!( + result, + Err(ProcessError::Timeout { + program: "test process", + duration, + }) if duration == Duration::from_millis(50) + )); + Ok(()) +} + +#[test] +fn process_child_parks_indefinitely() { + if env::var_os(PARKED_CHILD).is_none() { + return; + } + loop { + std::thread::park(); + } +} diff --git a/xtask/src/fuzz_campaign/execution.rs b/xtask/src/fuzz_campaign/execution.rs index ff3abdc..fc1fd92 100644 --- a/xtask/src/fuzz_campaign/execution.rs +++ b/xtask/src/fuzz_campaign/execution.rs @@ -41,7 +41,9 @@ impl CommandRunner for SystemRunner { replay(&output)?; Ok(output) } - OutputMode::Inherit => bounded_process::status(CARGO_FUZZ_PROCESS, &mut command), + OutputMode::Inherit => { + bounded_process::status(CARGO_FUZZ_PROCESS, &mut command, plan.deadline()) + } } } } From cdd8ebb745f13665e742405491036cce678e2244 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:08:19 -0700 Subject: [PATCH 026/113] Fix: revalidate the source root after scanning --- CHANGELOG.md | 6 +++--- xtask/src/source_structure.rs | 1 + xtask/tests/source_policy_contract.rs | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0106a14..6dfcbcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,9 +41,9 @@ after its public API and format compatibility policies are established. `b3sum`, and CI refuses Rust, Python, or shell source modules that exceed the documented 500-physical-line hard maximum, including test modules. - Repository source verification now uses capability-relative, no-follow file - opens and verifies repository-root identity after Git inventory, so a - persistent root replacement or source path replaced with a symlink is - refused before source bytes are read. The pure Rust boundary also refuses + opens and verifies repository-root identity after Git inventory and again + after source scanning, so a persistent root replacement or source path + replaced with a symlink is refused. The pure Rust boundary also refuses `.py`, `.pyw`, and extensionless executable Python shebangs. - The repository `cargo xtask` alias and Rust command contract are now explicitly silent on success and emit one typed `Error:` diagnostic with diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 60eb4f5..0906c85 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -34,6 +34,7 @@ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> let paths = source_paths(repository_root)?; verify_source_root(&source_root, repository_root)?; let violations = source_violations(&source_root, paths)?; + verify_source_root(&source_root, repository_root)?; if violations.is_empty() { Ok(()) } else { diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 5b488ba..d871bc8 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -20,3 +20,13 @@ fn repository_file_admission_declares_its_unix_scope() { assert!(REPOSITORY_FILE.contains("intentionally supported only on Unix hosts")); assert!(REPOSITORY_FILE.contains("Unix device and inode identity")); } + +#[test] +fn source_scan_revalidates_repository_identity_after_reading() { + assert_eq!( + SOURCE_STRUCTURE + .matches("verify_source_root(&source_root, repository_root)?;") + .count(), + 2 + ); +} From 1385df13a1e3a5d9e1abbe5af9945d3bc4ef591c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:10:31 -0700 Subject: [PATCH 027/113] Fix: preserve primary Git inventory failures --- CHANGELOG.md | 3 + xtask/src/git_inventory/error.rs | 11 +++ xtask/src/git_inventory/process.rs | 104 +++++++++++++++++------ xtask/src/git_inventory/process/tests.rs | 32 ++++++- 4 files changed, 124 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dfcbcc..3e4fe63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ after its public API and format compatibility policies are established. after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses `.py`, `.pyw`, and extensionless executable Python shebangs. +- Git path inventory failures now remain primary when child cleanup, waiting, + or diagnostic collection also fails; the secondary failure remains typed and + inspectable. - The repository `cargo xtask` alias and Rust command contract are now explicitly silent on success and emit one typed `Error:` diagnostic with exit status 1 on refusal; untrusted control characters are escaped so the diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs index 21017e5..a4a5c11 100644 --- a/xtask/src/git_inventory/error.rs +++ b/xtask/src/git_inventory/error.rs @@ -14,6 +14,10 @@ pub(crate) enum GitOutputUnit { } pub(crate) enum GitInventoryError { + Cleanup { + primary: Box, + cleanup: Box, + }, DuplicatePath(Vec), Failed { operation: &'static str, @@ -57,6 +61,12 @@ impl fmt::Debug for GitInventoryError { impl fmt::Display for GitInventoryError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Cleanup { primary, cleanup } => { + write!( + formatter, + "{primary}; additionally, cleanup failed: {cleanup}" + ) + } Self::DuplicatePath(path) => { formatter.write_str("git returned duplicate path `")?; escaped_bytes(formatter, path)?; @@ -117,6 +127,7 @@ impl GitOutputUnit { impl Error for GitInventoryError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { + Self::Cleanup { primary, .. } => Some(primary), Self::DiagnosticEncoding { source, .. } => Some(source), Self::Run { source, .. } => Some(source), Self::DuplicatePath(_) diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index 65475c3..e43b787 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -52,18 +52,18 @@ fn start_git( source, })?; let Some(stdout) = child.stdout.take() else { - cleanup_child(&mut child, operation)?; - return Err(GitInventoryError::Pipe { + let primary = GitInventoryError::Pipe { operation, stream: "stdout", - }); + }; + return Err(cleanup_child(&mut child, operation, primary)); }; let Some(stderr) = child.stderr.take() else { - cleanup_child(&mut child, operation)?; - return Err(GitInventoryError::Pipe { + let primary = GitInventoryError::Pipe { operation, stream: "stderr", - }); + }; + return Err(cleanup_child(&mut child, operation, primary)); }; let diagnostic_worker = thread::Builder::new() .name(String::from("xtask-git-diagnostic")) @@ -71,12 +71,12 @@ fn start_git( let diagnostic_worker = match diagnostic_worker { Ok(worker) => worker, Err(source) => { - cleanup_child(&mut child, operation)?; - return Err(GitInventoryError::Run { + let primary = GitInventoryError::Run { operation, action: "start the diagnostic reader for", source, - }); + }; + return Err(cleanup_child(&mut child, operation, primary)); } }; Ok(GitProcess { @@ -102,18 +102,28 @@ fn collect_git_result( action: "wait for", source, }); - let diagnostic = diagnostic_worker - .join() - .map_err(|_| GitInventoryError::Worker { operation })?; - - stop?; - let paths = paths?; - let status = status?; - let diagnostic = diagnostic.map_err(|source| GitInventoryError::Run { - operation, - action: "read diagnostics from", - source, - })?; + let diagnostic = diagnostic_worker.join(); + let paths = match paths { + Ok(paths) => paths, + Err(primary) => { + return Err(preserve_collection_failure( + primary, stop, status, diagnostic, operation, + )); + } + }; + let status = match status { + Ok(status) => status, + Err(primary) => { + return Err(preserve_diagnostic_failure(primary, diagnostic, operation)); + } + }; + let diagnostic = diagnostic + .map_err(|_| GitInventoryError::Worker { operation })? + .map_err(|source| GitInventoryError::Run { + operation, + action: "read diagnostics from", + source, + })?; if diagnostic.exceeded { return Err(GitInventoryError::OutputBound { operation, @@ -145,15 +155,61 @@ fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), GitInv }) } -fn cleanup_child(child: &mut Child, operation: &'static str) -> Result<(), GitInventoryError> { +fn cleanup_child( + child: &mut Child, + operation: &'static str, + primary: GitInventoryError, +) -> GitInventoryError { let stop = request_stop(child, operation); let wait = child.wait().map_err(|source| GitInventoryError::Run { operation, action: "wait for", source, }); - stop?; - wait.map(|_| ()) + let primary = preserve_error(primary, stop); + preserve_error(primary, wait.map(|_| ())) +} + +fn preserve_collection_failure( + primary: GitInventoryError, + stop: Result<(), GitInventoryError>, + status: Result, + diagnostic: thread::Result>, + operation: &'static str, +) -> GitInventoryError { + let primary = preserve_error(primary, stop); + let primary = preserve_error(primary, status.map(|_| ())); + preserve_diagnostic_failure(primary, diagnostic, operation) +} + +fn preserve_diagnostic_failure( + primary: GitInventoryError, + diagnostic: thread::Result>, + operation: &'static str, +) -> GitInventoryError { + let cleanup = match diagnostic { + Ok(Ok(_)) => return primary, + Ok(Err(source)) => GitInventoryError::Run { + operation, + action: "read diagnostics from", + source, + }, + Err(_) => GitInventoryError::Worker { operation }, + }; + preserve_error(primary, Err(cleanup)) +} + +fn preserve_error( + primary: GitInventoryError, + cleanup: Result<(), GitInventoryError>, +) -> GitInventoryError { + match cleanup { + Ok(()) => primary, + Err(cleanup) => GitInventoryError::Cleanup { + primary: Box::new(primary), + cleanup: Box::new(cleanup), + }, + } } fn git_failure( diff --git a/xtask/src/git_inventory/process/tests.rs b/xtask/src/git_inventory/process/tests.rs index 1cfb253..0b114cc 100644 --- a/xtask/src/git_inventory/process/tests.rs +++ b/xtask/src/git_inventory/process/tests.rs @@ -1,8 +1,8 @@ //! This module owns adversarial Git diagnostic-bound tests. -use std::io::Cursor; +use std::io::{self, Cursor}; -use super::{GitInventoryError, git_failure}; +use super::{GitInventoryError, git_failure, preserve_error}; use crate::process_output::bounded_bytes; #[test] @@ -26,3 +26,31 @@ fn git_diagnostics_are_drained_but_only_the_bound_is_retained() { Ok(ref diagnostic) if diagnostic.bytes == b"abc" && diagnostic.exceeded )); } + +#[test] +fn simultaneous_git_failures_preserve_the_detected_error() { + let primary = GitInventoryError::OutputFraming { + operation: "test inventory", + }; + let cleanup = GitInventoryError::Run { + operation: "test inventory", + action: "stop", + source: io::Error::other("cleanup failed"), + }; + + let error = preserve_error(primary, Err(cleanup)); + + assert!(matches!( + error, + GitInventoryError::Cleanup { + primary, + cleanup, + } if matches!(*primary, GitInventoryError::OutputFraming { + operation: "test inventory", + }) && matches!(*cleanup, GitInventoryError::Run { + operation: "test inventory", + action: "stop", + .. + }) + )); +} From 50b5f21e2dc613a1b97b545f1c048df4abb822b4 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:11:38 -0700 Subject: [PATCH 028/113] Fix: preserve Dependabot list boundaries --- CHANGELOG.md | 5 +++-- .../src/documentation_integrity/dependabot.rs | 18 ++++++++++-------- .../dependabot/tests.rs | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e4fe63..b726abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,9 @@ after its public API and format compatibility policies are established. boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow - commands as YAML, and applies one deadline across captured and inherited - child execution and output collection. + commands as YAML, preserves declarations after Dependabot directory lists, + and applies one deadline across captured and inherited child execution and + output collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index 198b945..803d2ab 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -87,17 +87,19 @@ fn block_scopes(block: &[&str]) -> Result, DocumentationErr .map(unquote) .ok_or_else(|| contract("every update block names an ecosystem"))?; let mut scopes = Vec::new(); - let mut lines = block.iter(); - while let Some(line) = lines.next() { + let mut remaining = block; + while let Some((line, rest)) = remaining.split_first() { + remaining = rest; if let Some(directory) = line.strip_prefix(" directory: ") { scopes.push(DependencyScope::new(ecosystem, unquote(directory))); } else if *line == " directories:" { - scopes.extend( - lines - .by_ref() - .map_while(|entry| entry.strip_prefix(" - ")) - .map(|directory| DependencyScope::new(ecosystem, unquote(directory))), - ); + while let Some((entry, rest)) = remaining.split_first() { + let Some(directory) = entry.strip_prefix(" - ") else { + break; + }; + scopes.push(DependencyScope::new(ecosystem, unquote(directory))); + remaining = rest; + } } } if scopes.is_empty() { diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs index 93d10bf..4de7395 100644 --- a/xtask/src/documentation_integrity/dependabot/tests.rs +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -41,6 +41,25 @@ fn complete_uniform_dependabot_policy_is_admitted() { assert!(super::admit(POLICY, &required()).is_ok()); } +#[test] +fn list_termination_preserves_the_following_scope_declaration() { + let block = [ + " - package-ecosystem: cargo", + " directories:", + " - /", + " directory: /xtask", + ]; + + let scopes = super::block_scopes(&block); + assert!(matches!( + scopes, + Ok(scopes) if scopes == vec![ + DependencyScope::new("cargo", "/"), + DependencyScope::new("cargo", "/xtask"), + ] + )); +} + #[test] fn missing_manifest_scope_is_refused() { let policy = POLICY.replace(" - /xtask\n", ""); From eaf0378b059c7250bdbec693ce7626251a41c9cd Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:14:02 -0700 Subject: [PATCH 029/113] Fix: bound documentation error formatting --- .../documentation_integrity/error/display.rs | 149 ++++++++++-------- xtask/tests/documentation_cli_contract.rs | 18 +++ 2 files changed, 105 insertions(+), 62 deletions(-) diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 95584de..f98f05a 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -40,76 +40,101 @@ impl fmt::Display for DocumentationError { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } Self::Process(error) => write!(formatter, "{error}"), - Self::RepositoryFileEncoding { path, .. } => { - write!(formatter, "repository file `{path}` is not UTF-8") - } - Self::RepositoryFileInspect { path, .. } => { - write!(formatter, "cannot inspect repository file `{path}`") - } - Self::RepositoryFileNonRegular(path) => { - write!(formatter, "repository file is not regular: `{path}`") - } - Self::RepositoryFileTooLarge { path, maximum } => write!( - formatter, - "repository file `{path}` exceeds the {maximum}-byte bound" - ), - Self::RepositoryContract { path, requirement } => { - write!( - formatter, - "repository file `{path}` violates: {requirement}" - ) - } - Self::RepositoryContractAt { - path, - subject, - requirement, - } => repository_contract_at(formatter, path, subject, requirement), - Self::RepositoryJson { path, .. } => { - write!(formatter, "repository file `{path}` is not valid JSON") - } - Self::RepositoryYaml { path, .. } => { - write!(formatter, "repository file `{path}` is not valid YAML") - } + error @ (Self::RepositoryFileEncoding { .. } + | Self::RepositoryFileInspect { .. } + | Self::RepositoryFileNonRegular(_) + | Self::RepositoryFileTooLarge { .. } + | Self::RepositoryContract { .. } + | Self::RepositoryContractAt { .. } + | Self::RepositoryJson { .. } + | Self::RepositoryYaml { .. } + | Self::RepositoryValue { .. }) => repository_file(formatter, error), Self::RepositoryRootChanged(path) => { repository_root(formatter, RepositoryRootDiagnostic::Changed, path) } Self::RepositoryRootInspect { path, .. } => { repository_root(formatter, RepositoryRootDiagnostic::Inspect, path) } - Self::RepositoryValue { - path, - field, - expected, - observed, - } => repository_value(formatter, path, field, expected, observed.as_deref()), - Self::VersionMismatch { - program, - expected, - observed, - } => write!( - formatter, - "{program} version mismatch: expected {expected:?}, observed {observed:?}" - ), - Self::ToolFailed { - program, - code, - stdout, - stderr, - } => tool_failed(formatter, program, *code, stdout, stderr), - Self::ToolOutputEncoding { - program, stream, .. - } => { - write!(formatter, "{program} {stream} is not UTF-8") - } - Self::ToolUnavailable { - program, - install_version, - .. - } => write!( + error @ (Self::VersionMismatch { .. } + | Self::ToolFailed { .. } + | Self::ToolOutputEncoding { .. } + | Self::ToolUnavailable { .. }) => tool(formatter, error), + } + } +} + +fn repository_file(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::RepositoryFileEncoding { path, .. } => { + write!(formatter, "repository file `{path}` is not UTF-8") + } + DocumentationError::RepositoryFileInspect { path, .. } => { + write!(formatter, "cannot inspect repository file `{path}`") + } + DocumentationError::RepositoryFileNonRegular(path) => { + write!(formatter, "repository file is not regular: `{path}`") + } + DocumentationError::RepositoryFileTooLarge { path, maximum } => write!( + formatter, + "repository file `{path}` exceeds the {maximum}-byte bound" + ), + DocumentationError::RepositoryContract { path, requirement } => { + write!( formatter, - "{program} is unavailable; install version {install_version}" - ), + "repository file `{path}` violates: {requirement}" + ) + } + DocumentationError::RepositoryContractAt { + path, + subject, + requirement, + } => repository_contract_at(formatter, path, subject, requirement), + DocumentationError::RepositoryJson { path, .. } => { + write!(formatter, "repository file `{path}` is not valid JSON") + } + DocumentationError::RepositoryYaml { path, .. } => { + write!(formatter, "repository file `{path}` is not valid YAML") + } + DocumentationError::RepositoryValue { + path, + field, + expected, + observed, + } => repository_value(formatter, path, field, expected, observed.as_deref()), + _ => Err(fmt::Error), + } +} + +fn tool(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::VersionMismatch { + program, + expected, + observed, + } => write!( + formatter, + "{program} version mismatch: expected {expected:?}, observed {observed:?}" + ), + DocumentationError::ToolFailed { + program, + code, + stdout, + stderr, + } => tool_failed(formatter, program, *code, stdout, stderr), + DocumentationError::ToolOutputEncoding { + program, stream, .. + } => { + write!(formatter, "{program} {stream} is not UTF-8") } + DocumentationError::ToolUnavailable { + program, + install_version, + .. + } => write!( + formatter, + "{program} is unavailable; install version {install_version}" + ), + _ => Err(fmt::Error), } } diff --git a/xtask/tests/documentation_cli_contract.rs b/xtask/tests/documentation_cli_contract.rs index d1a1c24..8d088a7 100644 --- a/xtask/tests/documentation_cli_contract.rs +++ b/xtask/tests/documentation_cli_contract.rs @@ -7,6 +7,24 @@ mod documentation_tools; use std::io; +const DOCUMENTATION_ERROR_DISPLAY: &str = + include_str!("../src/documentation_integrity/error/display.rs"); + +#[test] +fn documentation_error_formatter_stays_below_the_hard_function_limit() -> Result<(), &'static str> { + let (_, after_signature) = DOCUMENTATION_ERROR_DISPLAY + .split_once(" fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {") + .ok_or("display implementation must retain its formatter")?; + let (body, _) = after_signature + .split_once("\n }\n}\n\nfn ") + .ok_or("display formatter must remain a directly inspectable function")?; + assert!( + body.lines().count() <= 59, + "DocumentationError::fmt exceeds the 60-line hard limit" + ); + Ok(()) +} + #[test] fn successful_verification_runs_every_documentation_tool_silently() -> Result<(), io::Error> { let tools = documentation_tools::DocumentationTools::create()?; From b938b1accf1d7298a6c3b6bd26dcd74acfaef751 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:14:50 -0700 Subject: [PATCH 030/113] Fix: distinguish empty Git paths --- CHANGELOG.md | 3 ++- xtask/src/git_inventory/error.rs | 7 +++++++ xtask/src/git_inventory/path_stream.rs | 2 +- xtask/src/git_inventory/path_stream/tests.rs | 8 ++++++-- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b726abe..01f0426 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,8 @@ after its public API and format compatibility policies are established. `.py`, `.pyw`, and extensionless executable Python shebangs. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and - inspectable. + inspectable. Empty path records and unterminated path bytes produce distinct, + accurate typed diagnostics. - The repository `cargo xtask` alias and Rust command contract are now explicitly silent on success and emit one typed `Error:` diagnostic with exit status 1 on refusal; untrusted control characters are escaped so the diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs index a4a5c11..c2ce986 100644 --- a/xtask/src/git_inventory/error.rs +++ b/xtask/src/git_inventory/error.rs @@ -19,6 +19,9 @@ pub(crate) enum GitInventoryError { cleanup: Box, }, DuplicatePath(Vec), + EmptyPath { + operation: &'static str, + }, Failed { operation: &'static str, code: Option, @@ -72,6 +75,9 @@ impl fmt::Display for GitInventoryError { escaped_bytes(formatter, path)?; formatter.write_str("`") } + Self::EmptyPath { operation } => { + write!(formatter, "`{operation}` returned an empty path") + } Self::Failed { operation, code, @@ -131,6 +137,7 @@ impl Error for GitInventoryError { Self::DiagnosticEncoding { source, .. } => Some(source), Self::Run { source, .. } => Some(source), Self::DuplicatePath(_) + | Self::EmptyPath { .. } | Self::Failed { .. } | Self::OutputBound { .. } | Self::OutputFraming { .. } diff --git a/xtask/src/git_inventory/path_stream.rs b/xtask/src/git_inventory/path_stream.rs index a95c81a..eade544 100644 --- a/xtask/src/git_inventory/path_stream.rs +++ b/xtask/src/git_inventory/path_stream.rs @@ -130,7 +130,7 @@ impl GitPathDecoder { fn admit_path(&mut self) -> Result<(), GitInventoryError> { if self.current.is_empty() { - return Err(GitInventoryError::OutputFraming { + return Err(GitInventoryError::EmptyPath { operation: self.operation, }); } diff --git a/xtask/src/git_inventory/path_stream/tests.rs b/xtask/src/git_inventory/path_stream/tests.rs index 96eec42..d7a86f4 100644 --- a/xtask/src/git_inventory/path_stream/tests.rs +++ b/xtask/src/git_inventory/path_stream/tests.rs @@ -65,11 +65,15 @@ fn git_path_stream_refuses_empty_records() { for stream in [&b"\0"[..], &b"a\0\0"[..]] { let result = read_paths_with(Cursor::new(stream), "test paths", TEST_LIMITS); assert!(matches!( - result, - Err(GitInventoryError::OutputFraming { + &result, + Err(GitInventoryError::EmptyPath { operation: "test paths" }) )); + assert_eq!( + result.err().map(|error| error.to_string()), + Some(String::from("`test paths` returned an empty path")) + ); } } From efcd45e2b3abef3b30032b40ea8c89a33d6832fa Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:16:50 -0700 Subject: [PATCH 031/113] Fix: remove Rust stdout test output --- xtask/src/bounded_process/tests.rs | 46 +++++++++++++++------------ xtask/tests/source_policy_contract.rs | 6 ++++ 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index 4caddb5..88664db 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -1,29 +1,46 @@ //! This module owns bounded child-process regression evidence. use std::env; -use std::io::{self, Write}; +use std::fs; +use std::io; use std::process::{Command, Stdio}; +use std::str; use std::time::Duration; use super::{ProcessError, capture, status}; +use crate::test_directory::TestDirectory; -const OUTPUT_CHILD: &str = "KEEP_XTASK_BOUNDED_OUTPUT_CHILD"; const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_CHILD"; #[test] fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box> { - let executable = env::current_exe()?; - let mut command = Command::new(executable); + let repository = TestDirectory::create("bounded-process-output")?; + let blob = repository.path().join("oversized.bin"); + fs::write(&blob, vec![b'x'; 1_048_577])?; + let initialized = Command::new("git") + .args(["init", "--quiet"]) + .current_dir(repository.path()) + .status()?; + if !initialized.success() { + return Err(io::Error::other("cannot initialize fixture repository").into()); + } + let hashed = Command::new("git") + .args(["hash-object", "-w", "oversized.bin"]) + .current_dir(repository.path()) + .output()?; + if !hashed.status.success() { + return Err(io::Error::other("cannot hash fixture blob").into()); + } + let object_id = str::from_utf8(&hashed.stdout)?.trim(); + let mut command = Command::new("git"); command - .args([ - "--exact", - "bounded_process::tests::process_child_writes_excess_output", - ]) - .env(OUTPUT_CHILD, "1") + .args(["cat-file", "blob", object_id]) + .current_dir(repository.path()) .stdin(Stdio::null()); let result = capture("test process", &mut command, Some(Duration::from_secs(5))); + repository.close()?; assert!(matches!( result, @@ -36,17 +53,6 @@ fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box Result<(), io::Error> { - if env::var_os(OUTPUT_CHILD).is_none() { - return Ok(()); - } - let bytes = vec![b'x'; 1_048_577]; - let mut output = io::stdout().lock(); - output.write_all(&bytes)?; - output.flush() -} - #[test] fn inherited_process_obeys_the_process_deadline() -> Result<(), Box> { let executable = env::current_exe()?; diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index d871bc8..f65eb95 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -1,6 +1,7 @@ //! Written-policy regression evidence for the executable source-size law. const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); +const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); @@ -30,3 +31,8 @@ fn source_scan_revalidates_repository_identity_after_reading() { 2 ); } + +#[test] +fn process_fixtures_do_not_write_to_rust_stdout() { + assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); +} From 10320c5cede58223125ab7eeb63f765cce291a94 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:18:55 -0700 Subject: [PATCH 032/113] Document: define repository process boundaries --- xtask/src/bounded_process.rs | 15 ++++++ xtask/src/bounded_process/capture.rs | 6 +++ xtask/src/bounded_process/error.rs | 9 ++++ xtask/src/git_inventory/error.rs | 26 +++++---- xtask/src/git_inventory/path_stream.rs | 11 ++++ xtask/tests/source_policy_contract.rs | 74 ++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 9 deletions(-) diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 0444df9..60e1be2 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -17,13 +17,28 @@ use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use reader::ReaderWorker; +/// The completed child status and any output retained by the selected mode. +/// +/// Captured execution retains at most one mebibyte per output stream. +/// Inherited execution leaves both byte vectors empty because the child writes +/// directly to the parent's configured streams. pub(crate) struct ProcessOutput { + /// The platform exit code, or `None` when the process ended by signal. pub(crate) code: Option, + /// Whether the platform status represents successful termination. pub(crate) succeeded: bool, + /// Captured standard output, or an empty vector in inherited mode. pub(crate) stdout: Vec, + /// Captured standard error, or an empty vector in inherited mode. pub(crate) stderr: Vec, } +/// Runs a child synchronously while inheriting its configured output streams. +/// +/// The optional deadline bounds the complete wait. The child starts in a +/// dedicated process group so timeout or polling failure terminates descendants +/// before this function returns. Spawn, poll, wait, timeout, and cleanup +/// failures retain their typed [`ProcessError`] boundary. pub(crate) fn status( program: &'static str, command: &mut Command, diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 3dae7bf..0267f29 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -11,6 +11,12 @@ use crate::process_output::BoundedBytes; const OUTPUT_LIMIT: usize = 1_048_576; +/// Runs a child synchronously and captures bounded standard output and error. +/// +/// Each stream is drained concurrently and retains at most one mebibyte. The +/// optional deadline covers child execution and reader collection. Failures +/// terminate the child's dedicated process group, join both readers, and retain +/// the primary and cleanup errors in [`ProcessError`]. pub(crate) fn capture( program: &'static str, command: &mut Command, diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs index 3ef6641..5a35331 100644 --- a/xtask/src/bounded_process/error.rs +++ b/xtask/src/bounded_process/error.rs @@ -5,34 +5,42 @@ use std::fmt; use std::io; use std::time::Duration; +/// A typed failure from synchronous, bounded child-process execution. pub(crate) enum ProcessError { + /// Reader or cleanup collection found another failure after the primary one. Additional { primary: Box, additional: Box, }, + /// Process-group termination or child reaping failed after a primary error. Cleanup { primary: Box, action: &'static str, source: io::Error, }, + /// A named operating-system process action failed. Io { program: &'static str, action: &'static str, source: io::Error, }, + /// A child configured for capture did not expose the requested pipe. MissingStream { program: &'static str, stream: &'static str, }, + /// A captured stream exceeded its fixed retained-byte limit. OutputLimit { program: &'static str, stream: &'static str, maximum: usize, }, + /// A dedicated output-reader thread stopped by panicking. ReaderPanic { program: &'static str, stream: &'static str, }, + /// The complete child operation exceeded its admitted duration. Timeout { program: &'static str, duration: Duration, @@ -40,6 +48,7 @@ pub(crate) enum ProcessError { } impl ProcessError { + /// Reports whether the primary process I/O failure is executable absence. pub(crate) fn is_not_found(&self) -> bool { match self { Self::Additional { primary, .. } | Self::Cleanup { primary, .. } => { diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs index c2ce986..6e93bbe 100644 --- a/xtask/src/git_inventory/error.rs +++ b/xtask/src/git_inventory/error.rs @@ -8,51 +8,59 @@ use std::string::FromUtf8Error; use crate::diagnostic::escaped_controls; #[derive(Clone, Copy)] +/// The unit named by a bounded Git-output failure. pub(crate) enum GitOutputUnit { + /// A byte-count bound. Bytes, + /// A path-record count bound. Items, } +/// A typed failure while listing or decoding repository paths from Git. pub(crate) enum GitInventoryError { + /// Cleanup failed after an earlier inventory failure was already detected. Cleanup { primary: Box, cleanup: Box, }, + /// Git emitted the same path record more than once. DuplicatePath(Vec), - EmptyPath { - operation: &'static str, - }, + /// Git emitted an empty NUL-framed path record. + EmptyPath { operation: &'static str }, + /// Git exited unsuccessfully and returned valid UTF-8 diagnostics. Failed { operation: &'static str, code: Option, stderr: String, }, + /// Git exited unsuccessfully with diagnostics that were not UTF-8. DiagnosticEncoding { operation: &'static str, code: Option, source: FromUtf8Error, }, + /// A retained byte or item count exceeded its fixed bound. OutputBound { operation: &'static str, stream: &'static str, maximum: usize, unit: GitOutputUnit, }, - OutputFraming { - operation: &'static str, - }, + /// Git ended its output with bytes not terminated by a NUL delimiter. + OutputFraming { operation: &'static str }, + /// A Git child configured for capture did not expose a requested pipe. Pipe { operation: &'static str, stream: &'static str, }, + /// A named operating-system action for the Git child failed. Run { operation: &'static str, action: &'static str, source: io::Error, }, - Worker { - operation: &'static str, - }, + /// The concurrent diagnostic-reader thread stopped by panicking. + Worker { operation: &'static str }, } impl fmt::Debug for GitInventoryError { diff --git a/xtask/src/git_inventory/path_stream.rs b/xtask/src/git_inventory/path_stream.rs index eade544..a6f5311 100644 --- a/xtask/src/git_inventory/path_stream.rs +++ b/xtask/src/git_inventory/path_stream.rs @@ -18,6 +18,11 @@ struct GitPathLimits { paths: usize, } +/// Decodes Git's NUL-framed path output into deterministic bytewise order. +/// +/// The call blocks while reading and refuses empty, duplicate, unterminated, +/// oversized, or excessive path records. Individual paths are limited to 4,096 +/// bytes; the stream is limited to 16 MiB and 100,000 records. pub(super) fn read_paths( reader: impl Read, operation: &'static str, @@ -64,13 +69,19 @@ struct GitPathDecoder { } #[derive(Clone, Eq, Ord, PartialEq, PartialOrd)] +/// An opaque repository path admitted from Git's byte-oriented output. +/// +/// Ordering is bytewise and therefore independent of locale and filesystem +/// enumeration order. pub(crate) struct GitPath(Vec); impl GitPath { + /// Wraps path bytes that the stream decoder has already admitted. pub(crate) const fn new(bytes: Vec) -> Self { Self(bytes) } + /// Returns the exact repository-relative bytes reported by Git. pub(crate) fn as_bytes(&self) -> &[u8] { &self.0 } diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index f65eb95..388d9c1 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -1,7 +1,13 @@ //! Written-policy regression evidence for the executable source-size law. const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); +const BOUNDED_PROCESS: &str = include_str!("../src/bounded_process.rs"); +const BOUNDED_PROCESS_CAPTURE: &str = include_str!("../src/bounded_process/capture.rs"); +const BOUNDED_PROCESS_ERROR: &str = include_str!("../src/bounded_process/error.rs"); const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); +const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); +const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs"); +const GIT_PROCESS: &str = include_str!("../src/git_inventory/process.rs"); const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); @@ -36,3 +42,71 @@ fn source_scan_revalidates_repository_identity_after_reading() { fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); } + +#[test] +fn repository_process_boundaries_document_every_exported_contract() -> Result<(), String> { + require_docs( + BOUNDED_PROCESS, + &["pub(crate) struct ProcessOutput", "pub(crate) fn status("], + )?; + require_docs(BOUNDED_PROCESS_CAPTURE, &["pub(crate) fn capture("])?; + require_docs( + BOUNDED_PROCESS_ERROR, + &[ + "pub(crate) enum ProcessError", + " Additional {", + " Cleanup {", + " Io {", + " MissingStream {", + " OutputLimit {", + " ReaderPanic {", + " Timeout {", + " pub(crate) fn is_not_found(", + ], + )?; + require_docs( + GIT_INVENTORY_ERROR, + &[ + "pub(crate) enum GitOutputUnit", + " Bytes,", + " Items,", + "pub(crate) enum GitInventoryError", + " Cleanup {", + " DuplicatePath(", + " EmptyPath {", + " Failed {", + " DiagnosticEncoding {", + " OutputBound {", + " OutputFraming {", + " Pipe {", + " Run {", + " Worker {", + ], + )?; + require_docs( + GIT_PATH_STREAM, + &[ + "pub(super) fn read_paths(", + "pub(crate) struct GitPath(", + " pub(crate) const fn new(", + " pub(crate) fn as_bytes(", + ], + )?; + require_docs(GIT_PROCESS, &["pub(crate) fn paths("]) +} + +fn require_docs(source: &str, declarations: &[&str]) -> Result<(), String> { + for declaration in declarations { + let (before, _) = source + .split_once(declaration) + .ok_or_else(|| format!("missing declaration `{declaration}`"))?; + let documented = before + .lines() + .next_back() + .is_some_and(|line| line.trim_start().starts_with("///")); + if !documented { + return Err(format!("missing rustdoc for `{declaration}`")); + } + } + Ok(()) +} From eef07d8e60116d199869b5e6b1467d3e879778a9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:23:16 -0700 Subject: [PATCH 033/113] Fix: refuse non-string workflow commands --- CHANGELOG.md | 6 +++--- .../workflow_contract.rs | 17 ++++++++++++----- .../workflow_contract/tests.rs | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01f0426..5d27641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,9 +17,9 @@ after its public API and format compatibility policies are established. boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow - commands as YAML, preserves declarations after Dependabot directory lists, - and applies one deadline across captured and inherited child execution and - output collection. + commands as YAML, rejects non-string `run` values, preserves declarations + after Dependabot directory lists, and applies one deadline across captured + and inherited child execution and output collection. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 7f6603a..f117291 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -51,11 +51,18 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let Some(steps) = document["jobs"]["documentation"]["steps"].as_vec() else { return Err(contract("workflow defines documentation job steps")); }; - Ok(steps - .iter() - .filter_map(|step| step["run"].as_str()) - .map(|run| run.trim_end_matches('\n').to_owned()) - .collect()) + let mut runs = Vec::new(); + for step in steps { + let run = &step["run"]; + if run.is_badvalue() { + continue; + } + let Some(run) = run.as_str() else { + return Err(contract("documentation job run values are strings")); + }; + runs.push(run.trim_end_matches('\n').to_owned()); + } + Ok(runs) } fn runs_are_reviewed(runs: &[String]) -> bool { diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index f2e4517..61d498a 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -60,6 +60,21 @@ jobs: )); } +#[test] +fn non_string_run_values_are_refused() { + let workflow = WORKFLOW.replace( + " next-job:", + " - name: Invalid executable\n run: true\n next-job:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run values are strings", + }) + )); +} + #[test] fn unreviewed_python_executables_are_refused() { let workflow = WORKFLOW.replace( From f1ab6a9b8f02343bac1f7d11fbded3341306819e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:25:07 -0700 Subject: [PATCH 034/113] Fix: isolate process fixtures from host Git --- CHANGELOG.md | 3 ++- xtask/src/bounded_process/tests.rs | 28 +++++++++++++++++++++------ xtask/tests/source_policy_contract.rs | 15 ++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d27641..cef4122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,8 @@ after its public API and format compatibility policies are established. simultaneous Markdown and link failures, parses documentation workflow commands as YAML, rejects non-string `run` values, preserves declarations after Dependabot directory lists, and applies one deadline across captured - and inherited child execution and output collection. + and inherited child execution and output collection. Git-backed process + fixtures ignore system and global Git configuration. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index 88664db..5e2b262 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -17,26 +17,28 @@ fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box Result<(), Box Result { + let path = env::var_os("PATH").ok_or_else(|| io::Error::other("PATH is unavailable"))?; + let global_config = repository.path().join("global.gitconfig"); + let mut command = Command::new("git"); + command + .env_clear() + .env("PATH", path) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", global_config) + .env("LC_ALL", "C") + .current_dir(repository.path()); + Ok(command) +} + #[test] fn inherited_process_obeys_the_process_deadline() -> Result<(), Box> { let executable = env::current_exe()?; diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 388d9c1..49657d7 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -43,6 +43,21 @@ fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); } +#[test] +fn process_fixtures_isolate_git_from_host_configuration() { + assert_eq!( + BOUNDED_PROCESS_TESTS + .matches("Command::new(\"git\")") + .count(), + 1 + ); + assert!(BOUNDED_PROCESS_TESTS.contains(".env_clear()")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"PATH\"")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_NOSYSTEM\", \"1\")")); + assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_GLOBAL\"")); + assert!(BOUNDED_PROCESS_TESTS.contains("--template=")); +} + #[test] fn repository_process_boundaries_document_every_exported_contract() -> Result<(), String> { require_docs( From f480ea53ddc5b7314c7f7793d126a044fec3415c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:27:19 -0700 Subject: [PATCH 035/113] Fix: preserve fixture template path bytes --- CHANGELOG.md | 3 ++- xtask/src/bounded_process/tests.rs | 23 ++++++++++++++++++++++- xtask/tests/source_policy_contract.rs | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cef4122..67ce163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ after its public API and format compatibility policies are established. commands as YAML, rejects non-string `run` values, preserves declarations after Dependabot directory lists, and applies one deadline across captured and inherited child execution and output collection. Git-backed process - fixtures ignore system and global Git configuration. + fixtures ignore system and global Git configuration and preserve non-UTF-8 + template paths without lossy conversion. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/xtask/src/bounded_process/tests.rs b/xtask/src/bounded_process/tests.rs index 5e2b262..5913183 100644 --- a/xtask/src/bounded_process/tests.rs +++ b/xtask/src/bounded_process/tests.rs @@ -1,8 +1,10 @@ //! This module owns bounded child-process regression evidence. use std::env; +use std::ffi::OsString; use std::fs; use std::io; +use std::path::Path; use std::process::{Command, Stdio}; use std::str; use std::time::Duration; @@ -24,7 +26,7 @@ fn external_output_is_drained_but_refused_above_the_bound() -> Result<(), Box Result Ok(command) } +fn template_argument(template: &Path) -> OsString { + let mut argument = OsString::from("--template="); + argument.push(template); + argument +} + +#[test] +fn fixture_template_argument_preserves_non_utf8_paths() { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let template = OsString::from_vec(b"/tmp/non-utf8-\xff".to_vec()); + let argument = template_argument(Path::new(&template)); + + assert_eq!( + argument.as_os_str().as_bytes(), + b"--template=/tmp/non-utf8-\xff" + ); +} + #[test] fn inherited_process_obeys_the_process_deadline() -> Result<(), Box> { let executable = env::current_exe()?; diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 49657d7..2dd75d5 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -56,6 +56,7 @@ fn process_fixtures_isolate_git_from_host_configuration() { assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_NOSYSTEM\", \"1\")")); assert!(BOUNDED_PROCESS_TESTS.contains(".env(\"GIT_CONFIG_GLOBAL\"")); assert!(BOUNDED_PROCESS_TESTS.contains("--template=")); + assert!(!BOUNDED_PROCESS_TESTS.contains("template.display()")); } #[test] From 6f5e15c03d763641750624564e71bf5d00ec32fc Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:32:08 -0700 Subject: [PATCH 036/113] Fix: bound inherited fuzz processes --- CHANGELOG.md | 3 +++ fuzz/README.md | 7 +++++++ fuzz/campaign.env | 2 ++ xtask/src/fuzz_campaign.rs | 2 +- xtask/src/fuzz_campaign/command.rs | 16 +++++++++------ xtask/src/fuzz_campaign/command/tests.rs | 17 ++++++++++++--- xtask/src/fuzz_campaign/execution/tests.rs | 4 ++-- xtask/src/fuzz_campaign/policy.rs | 24 ++++++++++++++++++++++ xtask/src/fuzz_campaign/policy/error.rs | 5 +++++ xtask/src/fuzz_campaign/policy/syntax.rs | 4 +++- xtask/src/fuzz_campaign/policy/tests.rs | 2 ++ xtask/tests/cli_contract.rs | 2 ++ 12 files changed, 75 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67ce163..363efcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ after its public API and format compatibility policies are established. and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. +- Fuzz build and run plans now carry external process deadlines from the + reviewed campaign policy. Run deadlines use checked addition of the + exploration budget and process-grace interval before process-group execution. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/fuzz/README.md b/fuzz/README.md index e900d6c..3963786 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -12,6 +12,13 @@ versions and resource limits. `cargo xtask fuzz` parses that file without shell substitution and refuses missing, duplicate, unknown, malformed, or out-of-bound values. +The external `cargo fuzz build` process uses the reviewed build timeout. Each +`cargo fuzz run` process uses its profile's exploration budget plus the +reviewed process-grace interval; the addition is checked before execution. +Expiry terminates and reaps the complete child process group. The grace +interval covers Cargo startup and libFuzzer shutdown without enlarging +libFuzzer's exploration budget. + The Rust task compares `cargo fuzz list` with the checked-in target files, sorts the exact target names, bounds child-process output, and exercises every target even if an earlier target fails. Run: diff --git a/fuzz/campaign.env b/fuzz/campaign.env index b5025cd..326dbe8 100644 --- a/fuzz/campaign.env +++ b/fuzz/campaign.env @@ -8,9 +8,11 @@ FUZZ_MAX_INPUT_BYTES=1048576 FUZZ_RSS_LIMIT_MB=1024 # Campaign-specific exploration budgets. +FUZZ_BUILD_TIMEOUT_SECONDS=600 FUZZ_SMOKE_SECONDS_PER_TARGET=15 FUZZ_SCHEDULED_SECONDS_PER_TARGET=600 FUZZ_CMIN_SECONDS_PER_TARGET=120 +FUZZ_PROCESS_GRACE_SECONDS=60 # Retained derived-corpus bounds. FUZZ_CORPUS_MAX_FILES=20000 diff --git a/xtask/src/fuzz_campaign.rs b/xtask/src/fuzz_campaign.rs index 9c1bcde..9e0361d 100644 --- a/xtask/src/fuzz_campaign.rs +++ b/xtask/src/fuzz_campaign.rs @@ -111,7 +111,7 @@ fn execute_plans( let plans = target::registered(repository_root, policy)? .into_iter() .map(|target| CommandPlan::new(policy, operation, target)) - .collect::>(); + .collect::, _>>()?; execution::run(repository_root, name, &plans)?; Ok(()) } diff --git a/xtask/src/fuzz_campaign/command.rs b/xtask/src/fuzz_campaign/command.rs index 1771416..4deea8c 100644 --- a/xtask/src/fuzz_campaign/command.rs +++ b/xtask/src/fuzz_campaign/command.rs @@ -6,7 +6,7 @@ mod tests; use std::ffi::OsString; use std::time::Duration; -use super::policy::CampaignPolicy; +use super::policy::{CampaignPolicy, PolicyError}; use super::profile::CampaignProfile; use super::target::FuzzTarget; @@ -38,7 +38,7 @@ impl CommandPlan { policy: &CampaignPolicy, operation: CampaignOperation, target: FuzzTarget, - ) -> Self { + ) -> Result { let mut arguments = vec![ OsString::from(format!("+{}", policy.toolchain())), OsString::from("fuzz"), @@ -46,7 +46,7 @@ impl CommandPlan { let (deadline, output_mode, refused_output_marker) = match operation { CampaignOperation::Build => { arguments.extend([OsString::from("build"), OsString::from(target.as_str())]); - (None, OutputMode::Inherit, None) + (Some(policy.build_timeout()), OutputMode::Inherit, None) } CampaignOperation::Minimize => { arguments.extend([OsString::from("cmin"), OsString::from(target.as_str())]); @@ -60,16 +60,20 @@ impl CommandPlan { CampaignOperation::Run(profile) => { arguments.extend([OsString::from("run"), OsString::from(target.as_str())]); push_fuzzer_arguments(&mut arguments, policy.seconds_per_target(profile), policy); - (None, OutputMode::Inherit, None) + ( + Some(policy.run_timeout(profile)?), + OutputMode::Inherit, + None, + ) } }; - Self { + Ok(Self { target, arguments, deadline, output_mode, refused_output_marker, - } + }) } pub(super) const fn target(&self) -> &FuzzTarget { diff --git a/xtask/src/fuzz_campaign/command/tests.rs b/xtask/src/fuzz_campaign/command/tests.rs index eccf143..aab93c0 100644 --- a/xtask/src/fuzz_campaign/command/tests.rs +++ b/xtask/src/fuzz_campaign/command/tests.rs @@ -15,7 +15,7 @@ fn run_plan_preserves_every_reviewed_resource_bound() -> Result<(), Box Result<(), Box Result<(), Box> { + let policy = policy()?; + let target = FuzzTarget::admit("segment_format".to_owned())?; + let plan = CommandPlan::new(&policy, CampaignOperation::Build, target)?; + + assert_eq!(plan.deadline(), Some(Duration::from_mins(10))); assert_eq!(plan.output_mode(), OutputMode::Inherit); Ok(()) } @@ -40,7 +51,7 @@ fn run_plan_preserves_every_reviewed_resource_bound() -> Result<(), Box Result<(), Box> { let policy = policy()?; let target = FuzzTarget::admit("blob_hasher".to_owned())?; - let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target); + let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target)?; assert_eq!(plan.deadline(), Some(Duration::from_mins(2))); assert_eq!( plan.refused_output_marker(), diff --git a/xtask/src/fuzz_campaign/execution/tests.rs b/xtask/src/fuzz_campaign/execution/tests.rs index a25fec5..0497167 100644 --- a/xtask/src/fuzz_campaign/execution/tests.rs +++ b/xtask/src/fuzz_campaign/execution/tests.rs @@ -35,7 +35,7 @@ fn every_target_runs_after_an_earlier_failure() -> Result<(), Box> { fn swallowed_minimization_failure_is_refused() -> Result<(), Box> { let policy = policy()?; let target = FuzzTarget::admit("first".to_owned())?; - let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target); + let plan = CommandPlan::new(&policy, CampaignOperation::Minimize, target)?; let mut runner = ScriptedRunner::new([output(true, b"Failed to minimize corpus: signal 6", b"")]); let Err(error) = execute_all(Path::new("."), "corpus minimization", &[plan], &mut runner) @@ -69,7 +69,7 @@ fn plans(operation: CampaignOperation) -> Result, Box BTreeMap<&'static str, String> { [ ("CARGO_FUZZ_VERSION", self.cargo_fuzz_version.clone()), + ( + "FUZZ_BUILD_TIMEOUT_SECONDS", + self.build_timeout_seconds.to_string(), + ), ( "FUZZ_CMIN_SECONDS_PER_TARGET", self.cmin_seconds_per_target.to_string(), @@ -69,6 +76,10 @@ impl CampaignPolicy { self.input_timeout_seconds.to_string(), ), ("FUZZ_MAX_INPUT_BYTES", self.max_input_bytes.to_string()), + ( + "FUZZ_PROCESS_GRACE_SECONDS", + self.process_grace_seconds.to_string(), + ), ("FUZZ_RSS_LIMIT_MB", self.rss_limit_mb.to_string()), ( "FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS", @@ -96,6 +107,17 @@ impl CampaignPolicy { self.cmin_seconds_per_target } + pub(super) const fn build_timeout(&self) -> Duration { + Duration::from_secs(self.build_timeout_seconds) + } + + pub(super) fn run_timeout(&self, profile: CampaignProfile) -> Result { + self.seconds_per_target(profile) + .checked_add(self.process_grace_seconds) + .map(Duration::from_secs) + .ok_or(PolicyError::CampaignDeadline) + } + pub(super) const fn corpus_max_bytes(&self) -> u64 { self.corpus_max_bytes } @@ -124,8 +146,10 @@ impl CampaignPolicy { Ok(Self { cargo_fuzz_version, toolchain, + build_timeout_seconds: bounded(values, "FUZZ_BUILD_TIMEOUT_SECONDS", 60, 3_600)?, input_timeout_seconds: bounded(values, "FUZZ_INPUT_TIMEOUT_SECONDS", 1, 60)?, max_input_bytes: bounded(values, "FUZZ_MAX_INPUT_BYTES", 1, 1_048_576)?, + process_grace_seconds: bounded(values, "FUZZ_PROCESS_GRACE_SECONDS", 1, 600)?, rss_limit_mb: bounded(values, "FUZZ_RSS_LIMIT_MB", 128, 8_192)?, smoke_seconds_per_target: bounded(values, "FUZZ_SMOKE_SECONDS_PER_TARGET", 1, 60)?, scheduled_seconds_per_target: bounded( diff --git a/xtask/src/fuzz_campaign/policy/error.rs b/xtask/src/fuzz_campaign/policy/error.rs index f69af72..c08e745 100644 --- a/xtask/src/fuzz_campaign/policy/error.rs +++ b/xtask/src/fuzz_campaign/policy/error.rs @@ -14,6 +14,7 @@ pub(crate) enum PolicyError { maximum: u64, }, CampaignOrder, + CampaignDeadline, CorpusCapacity, InvalidInteger(&'static str), InvalidToolchain, @@ -51,6 +52,9 @@ impl fmt::Display for PolicyError { Self::CampaignOrder => { formatter.write_str("scheduled fuzzing must exceed the smoke budget") } + Self::CampaignDeadline => { + formatter.write_str("fuzz process deadline exceeds the duration range") + } Self::CorpusCapacity => { formatter.write_str("corpus bytes cannot be smaller than one input") } @@ -85,6 +89,7 @@ impl Error for PolicyError { match self { Self::Read { source, .. } => Some(source), Self::Bound { .. } + | Self::CampaignDeadline | Self::CampaignOrder | Self::CorpusCapacity | Self::InvalidInteger(_) diff --git a/xtask/src/fuzz_campaign/policy/syntax.rs b/xtask/src/fuzz_campaign/policy/syntax.rs index 7a5162c..f021a8f 100644 --- a/xtask/src/fuzz_campaign/policy/syntax.rs +++ b/xtask/src/fuzz_campaign/policy/syntax.rs @@ -4,14 +4,16 @@ use std::collections::BTreeMap; use super::PolicyError; -const EXPECTED_KEYS: [&str; 13] = [ +const EXPECTED_KEYS: [&str; 15] = [ "CARGO_FUZZ_VERSION", + "FUZZ_BUILD_TIMEOUT_SECONDS", "FUZZ_CMIN_SECONDS_PER_TARGET", "FUZZ_CORPUS_MAX_BYTES", "FUZZ_CORPUS_MAX_FILES", "FUZZ_CORPUS_RETENTION_DAYS", "FUZZ_INPUT_TIMEOUT_SECONDS", "FUZZ_MAX_INPUT_BYTES", + "FUZZ_PROCESS_GRACE_SECONDS", "FUZZ_RSS_LIMIT_MB", "FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS", "FUZZ_SCHEDULED_SECONDS_PER_TARGET", diff --git a/xtask/src/fuzz_campaign/policy/tests.rs b/xtask/src/fuzz_campaign/policy/tests.rs index 4d2cbfb..6da00d0 100644 --- a/xtask/src/fuzz_campaign/policy/tests.rs +++ b/xtask/src/fuzz_campaign/policy/tests.rs @@ -9,9 +9,11 @@ fn repository_policy_preserves_every_reviewed_runtime_bound() -> Result<(), Poli let policy = CampaignPolicy::parse(POLICY)?; assert_eq!(policy.cargo_fuzz_version, "0.13.2"); assert_eq!(policy.toolchain, "nightly-2026-07-24"); + assert_eq!(policy.build_timeout_seconds, 600); assert_eq!(policy.smoke_seconds_per_target, 15); assert_eq!(policy.scheduled_seconds_per_target, 600); assert_eq!(policy.cmin_seconds_per_target, 120); + assert_eq!(policy.process_grace_seconds, 60); assert_eq!(policy.input_timeout_seconds, 5); assert_eq!(policy.max_input_bytes, 1_048_576); assert_eq!(policy.rss_limit_mb, 1_024); diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index d303086..528ed7e 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -61,12 +61,14 @@ fn fuzz_description_emits_the_admitted_smoke_policy() -> Result<(), io::Error> { assert_eq!( output.stdout, b"CARGO_FUZZ_VERSION: 0.13.2\n\ + FUZZ_BUILD_TIMEOUT_SECONDS: 600\n\ FUZZ_CMIN_SECONDS_PER_TARGET: 120\n\ FUZZ_CORPUS_MAX_BYTES: 536870912\n\ FUZZ_CORPUS_MAX_FILES: 20000\n\ FUZZ_CORPUS_RETENTION_DAYS: 14\n\ FUZZ_INPUT_TIMEOUT_SECONDS: 5\n\ FUZZ_MAX_INPUT_BYTES: 1048576\n\ + FUZZ_PROCESS_GRACE_SECONDS: 60\n\ FUZZ_RSS_LIMIT_MB: 1024\n\ FUZZ_SCHEDULED_FAILURE_RETENTION_DAYS: 30\n\ FUZZ_SECONDS_PER_TARGET: 15\n\ From 6e172542a20612769e6f09e21470b74ad47a2882 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:41:51 -0700 Subject: [PATCH 037/113] Fix: refuse attached Python env interpreters --- CHANGELOG.md | 3 ++- xtask/src/source_structure/python_source.rs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 363efcb..bf04b58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,8 @@ after its public API and format compatibility policies are established. opens and verifies repository-root identity after Git inventory and again after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses - `.py`, `.pyw`, and extensionless executable Python shebangs. + `.py`, `.pyw`, and extensionless executable Python shebangs, including + attached `env -S` interpreter strings. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index ee23236..3f60191 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -74,6 +74,9 @@ fn environment_word_selects_python(word: &[u8]) -> bool { if let Some(split) = word.strip_prefix(b"--split-string=") { return is_python_program(split); } + if let Some(split) = word.strip_prefix(b"-S").filter(|split| !split.is_empty()) { + return is_python_program(split); + } !word.starts_with(b"-") && !word.contains(&b'=') && is_python_program(word) } @@ -110,6 +113,8 @@ mod tests { b"#! /usr/bin/env python3 -I\n", b"#!/usr/bin/env -S python3 -I\n", b"#!/usr/bin/env -S \"python3 -I\"\n", + b"#!/usr/bin/env -Spython3 -I\n", + b"#!/usr/bin/env -S/opt/PyPy3 -I\n", b"#!/usr/bin/env --split-string=python3\n", b"#!/opt/PyPy3\n", ] { From 4914b6988971c48e7a9a2f44098360c7ead515a2 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:43:06 -0700 Subject: [PATCH 038/113] Fix: inspect every executable source shebang --- CHANGELOG.md | 4 +-- xtask/src/source_structure.rs | 2 +- xtask/src/source_structure/pure_rust_tests.rs | 26 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf04b58..dfbbb94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,8 +50,8 @@ after its public API and format compatibility policies are established. opens and verifies repository-root identity after Git inventory and again after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses - `.py`, `.pyw`, and extensionless executable Python shebangs, including - attached `env -S` interpreter strings. + `.py`, `.pyw`, and Python shebangs in every executable source candidate, + including attached `env -S` interpreter strings. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 0906c85..4402908 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -110,8 +110,8 @@ fn source_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { + refuse_executable_python(source_root, &relative)?; if is_extensionless_file(relative.as_str().as_bytes()) { - refuse_executable_python(source_root, &relative)?; continue; } let lines = source_line_count(source_root, &relative)?; diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 77ec43b..2b34b57 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -50,6 +50,32 @@ fn extensionless_executable_python_is_refused_by_the_pure_rust_boundary() Ok(()) } +#[test] +fn extension_bearing_executable_python_is_refused_by_the_pure_rust_boundary() +-> Result<(), Box> { + let directory = TestDirectory::create("extension-bearing-python")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check.sh"); + fs::write(&script, b"#!/usr/bin/python3\nprint('forbidden')\n")?; + let mut permissions = fs::metadata(&script)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&script, permissions)?; + let present = BTreeSet::from([GitPath::new(b"check.sh".to_vec())]); + + let paths = super::select_source_paths(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let result = super::source_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) if path == "check.sh" + )); + drop(source_root); + directory.close()?; + Ok(()) +} + #[test] fn extensionless_nonexecutable_text_is_not_a_source_module() -> Result<(), Box> { From 0922e4538ff2ffd7a14ebf5546262fe270ce1aa5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 14:44:32 -0700 Subject: [PATCH 039/113] Fix: refuse guarded documentation commands --- CHANGELOG.md | 10 +++++----- docs/dependencies/documentation-toolchain.md | 10 +++++----- .../documentation_integrity/workflow_contract.rs | 3 +++ .../workflow_contract/tests.rs | 15 +++++++++++++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfbbb94..90b8fb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,11 @@ after its public API and format compatibility policies are established. boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow - commands as YAML, rejects non-string `run` values, preserves declarations - after Dependabot directory lists, and applies one deadline across captured - and inherited child execution and output collection. Git-backed process - fixtures ignore system and global Git configuration and preserve non-UTF-8 - template paths without lossy conversion. + commands as YAML, rejects guarded or non-string `run` values, preserves + declarations after Dependabot directory lists, and applies one deadline + across captured and inherited child execution and output collection. + Git-backed process fixtures ignore system and global Git configuration and + preserve non-UTF-8 template paths without lossy conversion. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index b2248e4..2a20394 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -35,11 +35,11 @@ The Rust `cargo xtask documentation-integrity-check` boundary verifies the structure and exact BLAKE3 digest of the committed Node lock artifact and the exact BLAKE3 digest of the reviewed installer. The byte-exact lock admission refuses altered, omitted, or additional package records. The boundary parses -the CI workflow as YAML and admits only reviewed `run` fields from the -documentation job, then verifies each executable's reported version before -admitting its output as evidence. A missing tool, changed archive, unexpected -version, empty input corpus, unreviewed command, or tool failure refuses the -check. +the CI workflow as YAML and admits only reviewed, unguarded `run` fields from +the documentation job, then verifies each executable's reported version +before admitting its output as evidence. A missing tool, changed archive, +unexpected version, empty input corpus, guarded or unreviewed command, or tool +failure refuses the check. ## Determinism and network posture diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index f117291..9ec27c5 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -57,6 +57,9 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> if run.is_badvalue() { continue; } + if !step["if"].is_badvalue() { + return Err(contract("documentation job run steps are unguarded")); + } let Some(run) = run.as_str() else { return Err(contract("documentation job run values are strings")); }; diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 61d498a..0a4babf 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -75,6 +75,21 @@ fn non_string_run_values_are_refused() { )); } +#[test] +fn guarded_required_commands_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n if: false\n run:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run steps are unguarded", + }) + )); +} + #[test] fn unreviewed_python_executables_are_refused() { let workflow = WORKFLOW.replace( From 055d92ccd4f37b25a3d926328abccf6b84efa148 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:09:51 -0700 Subject: [PATCH 040/113] Fix: bind documentation tools to opened root --- .github/dependabot.yml | 1 + CHANGELOG.md | 5 +- Cargo.lock | 8 ++ Cargo.toml | 4 +- docs/Rust Standards.md | 28 +++++-- ...escriptor-bound-child-working-directory.md | 71 +++++++++++++++++ docs/adr/README.md | 1 + .../cap-std-and-cap-fs-ext-4.0.2.md | 23 ++++-- docs/dependencies/documentation-toolchain.md | 7 ++ repository-process-spawn/Cargo.toml | 13 ++++ repository-process-spawn/src/lib.rs | 34 +++++++++ .../tests/working_directory.rs | 76 +++++++++++++++++++ xtask/Cargo.toml | 3 + xtask/src/bounded_process.rs | 2 +- xtask/src/bounded_process/capture.rs | 19 ++++- xtask/src/documentation_integrity.rs | 14 +++- xtask/src/documentation_integrity/corpus.rs | 67 +++++++++------- .../documentation_integrity/corpus/tests.rs | 54 +++++++++++-- .../src/documentation_integrity/dependabot.rs | 7 +- .../dependabot/manifest.rs | 11 ++- .../dependabot/tests.rs | 3 +- .../src/documentation_integrity/execution.rs | 41 +++++----- .../execution/external_tests.rs | 9 ++- xtask/src/git_inventory.rs | 2 +- xtask/src/git_inventory/process.rs | 31 +++++--- xtask/src/repository_file.rs | 29 +++++++ 26 files changed, 462 insertions(+), 101 deletions(-) create mode 100644 docs/adr/0006-descriptor-bound-child-working-directory.md create mode 100644 repository-process-spawn/Cargo.toml create mode 100644 repository-process-spawn/src/lib.rs create mode 100644 repository-process-spawn/tests/working_directory.rs diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3e193fc..4dd473b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: - / - /benchmark - /fuzz + - /repository-process-spawn - /xtask schedule: interval: weekly diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b8fb7..959e732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,10 @@ after its public API and format compatibility policies are established. declarations after Dependabot directory lists, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and - preserve non-UTF-8 template paths without lossy conversion. + preserve non-UTF-8 template paths without lossy conversion. Documentation + Git inventory and tools start from one retained repository directory handle, + so transient replacement of the ambient repository path cannot redirect + validation. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/Cargo.lock b/Cargo.lock index b0ac41a..fbded8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -401,6 +401,13 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" +[[package]] +name = "repository-process-spawn" +version = "0.0.0" +dependencies = [ + "rustix", +] + [[package]] name = "rustix" version = "1.1.4" @@ -708,6 +715,7 @@ dependencies = [ "cap-fs-ext", "cap-std", "md-5", + "repository-process-spawn", "rustix", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 397b698..7049a72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,14 +24,14 @@ name = "streaming_cdc" harness = false [workspace] -members = [".", "benchmark", "xtask"] +members = [".", "benchmark", "repository-process-spawn", "xtask"] resolver = "3" [lints] workspace = true [workspace.lints.rust] -unsafe_code = "forbid" +unsafe_code = "deny" missing_docs = "deny" unused_must_use = "deny" unreachable_pub = "deny" diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 33da1ea..2d666be 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -142,7 +142,7 @@ All crates MUST inherit workspace lints. # Cargo.toml [workspace.lints.rust] -unsafe_code = "forbid" +unsafe_code = "deny" missing_docs = "deny" unused_must_use = "deny" unreachable_pub = "deny" @@ -204,7 +204,13 @@ Each crate: #![warn(clippy::cargo)] ``` -Clippy’s `pedantic` group is explicitly aggressive and can produce false positives; that is acceptable here. Exceptions must be local and justified rather than weakening the workspace globally. (⁠[Rust Docs](https://doc.rust-lang.org/stable/clippy/lints.html?utm_source=chatgpt.com)) +Ordinary crates strengthen the workspace's `unsafe_code = "deny"` to +`forbid`. A dedicated unsafe-boundary crate MAY carry a crate-level, +reason-bearing allowance only after satisfying §4.2. + +Clippy’s `pedantic` group is explicitly aggressive and can produce false +positives; that is acceptable here. Exceptions must be local and justified +rather than weakening the workspace globally. ### **3.2 No broad lint suppression** @@ -271,7 +277,8 @@ cargo clippy \ ### **4.1 Default rule** -`unsafe` is forbidden throughout Keep V1. +`unsafe` is forbidden throughout Keep V1 except inside a dedicated crate +admitted under §4.2. ```rust #![forbid(unsafe_code)] @@ -288,9 +295,10 @@ Do not permit unsafe merely because storage engines often eventually use: Earn it later. -### **4.2 Future unsafe admission** +### **4.2 Unsafe admission** -If unsafe becomes demonstrably necessary, it MUST live in a dedicated crate such as: +If unsafe becomes demonstrably necessary, it MUST live in a dedicated crate +such as: ```text keep-platform @@ -306,12 +314,20 @@ That crate MUST contain: - property tests around the safe wrapper; - platform-specific integration tests; - a written alternative analysis; -- benchmarks proving the unsafe implementation earns its existence. +- measurements proving that safe alternatives cannot establish the required + behavior; +- benchmarks when performance is part of the justification. Every unsafe block MUST have a nearby `SAFETY:` explanation proving all preconditions. “Required for performance” is not a proof. +The only current admission is +[`repository-process-spawn`](../repository-process-spawn/src/lib.rs), governed +by the +[descriptor-bound child working-directory decision](adr/0006-descriptor-bound-child-working-directory.md). +Its one hook calls only POSIX async-signal-safe `fchdir` between fork and exec. + --- ## **5. Repository and Crate Structure** diff --git a/docs/adr/0006-descriptor-bound-child-working-directory.md b/docs/adr/0006-descriptor-bound-child-working-directory.md new file mode 100644 index 0000000..a550d5e --- /dev/null +++ b/docs/adr/0006-descriptor-bound-child-working-directory.md @@ -0,0 +1,71 @@ +# ADR-0006: Descriptor-Bound Child Working Directory + +- Status: Accepted +- Date: 2026-07-28 +- Owners: Keep repository verification +- Related issue: none — review remediation for pull request 61 +- Depends on: ADR-0004 + +## Context + +Documentation verification inventories Git paths and runs pinned validation +tools. Opening the repository as a capability protects file reads, but passing +its ambient pathname to a child creates another replacement window. An +attacker could move the admitted directory, substitute another repository +while the tools run, and restore the original before the final identity check. + +The standard library accepts only a pathname for +`std::process::Command::current_dir`. On macOS, an open directory exposed +through `/dev/fd` cannot be used as that pathname. Changing the parent process +directory through the safe Rustix `fchdir` wrapper was measured and rejected: +parallel xtask tests observed the temporary global state and failed. + +The remaining standard-library hook is +`std::os::unix::process::CommandExt::pre_exec`, which is unsafe because code +between fork and exec must obey strict rules. + +## Decision + +Keep isolates the hook in the private `repository-process-spawn` workspace +crate. The crate admits one operation: + +1. Own a close-on-exec duplicate of the admitted repository directory. +2. Register a child setup hook that calls only Rustix `fchdir`. +3. Spawn through the existing bounded process adapters. + +POSIX specifies `fchdir` as async-signal-safe. The hook performs no allocation, +locking, buffered I/O, ambient path lookup, or user callback. The descriptor +closes on successful exec. A setup failure is returned by `Command::spawn`. + +The workspace denies unsafe code by default. Only the dedicated crate carries +an explained `unsafe_code` allowance. It contains no storage, identity, format, +network, or application policy. + +## Alternatives considered + +- Rechecking the repository pathname before and after tool execution does not + detect a transient substitution. +- `/dev/fd/` and `/proc/self/fd/` are not a portable child working + directory. The former is not traversable as a directory on macOS, and the + latter is not available there. +- Changing the parent directory under a mutex still mutates process-global + state visible to threads outside that mutex. +- Copying the repository into a temporary tree changes Git, ignore, link, and + untracked-file semantics and creates an unbounded materialization. +- Reimplementing subprocess management would duplicate the standard library's + file-descriptor, environment, signal, and error handling. + +## Consequences + +Git inventory and documentation tools start in the exact opened repository +even if its pathname is replaced. Parent process state remains unchanged, so +parallel tests and readers are deterministic. + +The boundary is Unix-specific and deliberately narrow. Any additional unsafe +operation, child hook, captured state, or consumer requires a new decision and +new executable evidence. The crate's regression test replaces the ambient +path, proves the child reads the retained directory, and proves the parent +working directory is unchanged. + +No durable or authoritative state is written. Failure refuses the repository +task, so no recovery protocol is required. diff --git a/docs/adr/README.md b/docs/adr/README.md index e2553f1..622275a 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -57,3 +57,4 @@ encryption, concurrency, or public-API surface it governs. - [ADR-0003: Deterministic content-defined chunking profiles](0003-deterministic-content-defined-chunking-profiles.md) - [ADR-0004: Hexagonal boundary architecture](0004-hexagonal-boundary-architecture.md) - [ADR-0005: Durable segment store protocol](0005-durable-segment-store-protocol.md) +- [ADR-0006: Descriptor-bound child working directory](0006-descriptor-bound-child-working-directory.md) diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 181bfce..24a3ab3 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -30,6 +30,16 @@ The bounded subprocess adapter uses Rustix's safe process API to send collection failure. This prevents descendants that inherited an output pipe from surviving the failed repository task. +Documentation verification also duplicates the admitted repository directory +handle and uses the isolated `repository-process-spawn` crate to start Git and +validation tools from that exact directory. Its child-only setup hook performs +one POSIX async-signal-safe `fchdir` after fork and before exec. Parent process +state never changes. A transient ambient-path replacement therefore cannot +redirect corpus inventory or tool execution. +The +[descriptor-bound child working-directory decision](../adr/0006-descriptor-bound-child-working-directory.md) +records the unsafe-boundary invariants and rejected alternatives. + ## Why the standard library is insufficient Checking a path and then reopening it with `std::fs` leaves a @@ -40,9 +50,9 @@ the admitted object. The standard library also has no portable API that combines capability-relative opens with no-follow and nonblocking semantics. Recreating that boundary locally would require operating-system-specific flags, handle -conversion, and path-resolution code. That would exceed a small local helper, -duplicate security-sensitive upstream work, and require unsafe code that Keep -otherwise forbids. +conversion, descriptor-relative directory changes, and path-resolution code. +That would exceed a small local helper, duplicate security-sensitive upstream +work, and require unsafe code that Keep otherwise forbids. ## Features and resolved graph @@ -90,9 +100,10 @@ dependencies. ## Failure and recovery boundaries -An open, metadata, or read failure is a typed refusal. The task never repairs, -rewrites, or substitutes repository data. Retained handles exist only for one -verification process and carry no durability or recovery semantics. +An open, metadata, read, descriptor-duplication, child-directory setup, or +child-spawn failure is a typed refusal. The task never repairs, rewrites, or +substitutes repository data. Retained handles exist only for one verification +process and carry no durability or recovery semantics. Keep can remove these dependencies without changing public or durable behavior by replacing them with an equally portable, safe implementation that preserves diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index 2a20394..d1e07c4 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -51,6 +51,13 @@ deletions; Git-trackable nonregular paths such as symlinks and tracked paths replaced by FIFOs are refused. Non-trackable special files cannot enter the Git-selected corpus. +Git inventory and each validation tool start through one retained repository +directory handle. Child-only setup changes directory through that handle after +fork and before exec; the parent working directory does not change. Replacing +the configured repository path, running checks against a substitute, and +restoring the original path cannot redirect either corpus selection or +validation. + The workflow checker disables `actionlint`'s optional `shellcheck` and `pyflakes` integrations. Neither auxiliary executable is admitted or pinned by this toolchain, so ambient PATH contents cannot expand the validation boundary. diff --git a/repository-process-spawn/Cargo.toml b/repository-process-spawn/Cargo.toml new file mode 100644 index 0000000..1e6a5e4 --- /dev/null +++ b/repository-process-spawn/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "repository-process-spawn" +version = "0.0.0" +edition = "2024" +rust-version = "1.96" +license = "Apache-2.0" +publish = false + +[dependencies] +rustix = { version = "=1.1.4", default-features = false, features = ["process", "std"] } + +[lints] +workspace = true diff --git a/repository-process-spawn/src/lib.rs b/repository-process-spawn/src/lib.rs new file mode 100644 index 0000000..978492d --- /dev/null +++ b/repository-process-spawn/src/lib.rs @@ -0,0 +1,34 @@ +//! Isolated child-process setup for descriptor-bound working directories. +//! +//! This crate contains Keep's only admitted unsafe boundary. It exists because +//! [`std::process::Command`] exposes child setup through an unsafe hook, while +//! an opened directory descriptor is required to prevent pathname-replacement +//! races. No filesystem, storage, identity, or durable-format code belongs +//! here. + +#![allow( + unsafe_code, + reason = "this dedicated crate owns the reviewed pre-exec fchdir boundary" +)] + +use std::os::fd::OwnedFd; +use std::os::unix::process::CommandExt; +use std::process::Command; + +/// Configures `command` to enter `directory` after fork and before exec. +/// +/// The descriptor remains owned by the command hook. Any `fchdir` failure is +/// returned by the later spawn operation. The hook performs no allocation, +/// locking, I/O buffering, or user callback. +pub fn set_working_directory(command: &mut Command, directory: OwnedFd) { + // SAFETY: POSIX specifies fchdir as async-signal-safe. The hook captures an + // owned descriptor and performs exactly that operation between fork and + // exec. It does not allocate, lock, inspect ambient paths, or call user + // code. Rustix supplies the checked safe wrapper around the system call. + unsafe { + command.pre_exec(move || { + rustix::process::fchdir(&directory)?; + Ok(()) + }); + } +} diff --git a/repository-process-spawn/tests/working_directory.rs b/repository-process-spawn/tests/working_directory.rs new file mode 100644 index 0000000..46d291b --- /dev/null +++ b/repository-process-spawn/tests/working_directory.rs @@ -0,0 +1,76 @@ +//! Integration laws for descriptor-bound child working directories. + +use std::env; +use std::fs::{self, File}; +use std::io; +use std::os::fd::OwnedFd; +use std::path::PathBuf; +use std::process::{self, Command}; + +use repository_process_spawn::set_working_directory; + +#[test] +fn child_uses_the_opened_directory_without_mutating_parent_state() +-> Result<(), Box> { + let parent = env::current_dir()?; + let world = test_world("replacement"); + let root = world.join("repository"); + let retained = world.join("retained"); + fs::create_dir_all(&root)?; + fs::write(root.join("marker"), b"original\n")?; + let directory: OwnedFd = File::open(&root)?.into(); + + fs::rename(&root, &retained)?; + fs::create_dir(&root)?; + fs::write(root.join("marker"), b"substitute\n")?; + + let mut command = Command::new("/bin/cat"); + command.arg("marker"); + set_working_directory(&mut command, directory); + let output = command.output()?; + + fs::remove_dir_all(&root)?; + fs::remove_dir_all(&retained)?; + fs::remove_dir(&world)?; + + assert!(output.status.success()); + assert_eq!(output.stdout, b"original\n"); + assert_eq!(env::current_dir()?, parent); + Ok(()) +} + +#[test] +fn non_directory_descriptor_refuses_before_exec() -> Result<(), Box> { + let world = test_world("non-directory"); + fs::create_dir(&world)?; + let file_path = world.join("file"); + fs::write(&file_path, b"not a directory\n")?; + let descriptor: OwnedFd = File::open(&file_path)?.into(); + let mut command = Command::new("/bin/true"); + set_working_directory(&mut command, descriptor); + + let result = command.spawn(); + fs::remove_file(&file_path)?; + fs::remove_dir(&world)?; + let error = match result { + Ok(mut child) => { + let termination = child.kill(); + let reap = child.wait(); + return Err(format!( + "non-directory descriptor unexpectedly reached exec; termination: {termination:?}; reap: {reap:?}" + ) + .into()); + } + Err(error) => error, + }; + + assert_eq!(error.kind(), io::ErrorKind::NotADirectory); + Ok(()) +} + +fn test_world(case: &str) -> PathBuf { + env::temp_dir().join(format!( + "keep-repository-process-spawn-{}-{case}", + process::id() + )) +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 4d437ab..854dec0 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -14,6 +14,7 @@ repository-tasks = [ "dep:cap-fs-ext", "dep:cap-std", "dep:md-5", + "dep:repository-process-spawn", "dep:rustix", "dep:serde", "dep:serde_json", @@ -28,6 +29,8 @@ cap-fs-ext = { version = "=4.0.2", default-features = false, features = ["std"], cap-std = { version = "=4.0.2", default-features = false, optional = true } # Pure Rust MD5 regenerates the public Gear-table recipe; it is not an identity primitive. md-5 = { version = "=0.11.0", default-features = false, optional = true } +# Dedicated unsafe boundary sets exact child working directories by descriptor. +repository-process-spawn = { path = "../repository-process-spawn", optional = true } # Safe POSIX process-group signaling bounds failed repository-tool subprocesses. rustix = { version = "=1.1.4", default-features = false, features = ["process", "std"], optional = true } # Serde drives duplicate-refusing repository JSON admission; no types escape xtask. diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 60e1be2..a2f0240 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -11,8 +11,8 @@ use std::os::unix::process::CommandExt; use std::process::Command; use std::time::Duration; -pub(crate) use capture::capture; use capture::wait_for_child; +pub(crate) use capture::{capture, capture_with}; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use reader::ReaderWorker; diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 0267f29..56d19f7 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -21,9 +21,18 @@ pub(crate) fn capture( program: &'static str, command: &mut Command, deadline: Option, +) -> Result { + capture_with(program, command, deadline, Command::spawn) +} + +pub(crate) fn capture_with( + program: &'static str, + command: &mut Command, + deadline: Option, + spawn: impl FnOnce(&mut Command) -> Result, ) -> Result { let deadline = ProcessDeadline::new(program, deadline)?; - CapturedProcess::start(program, command)?.finish(program, &deadline) + CapturedProcess::start(program, command, spawn)?.finish(program, &deadline) } struct CapturedProcess { @@ -33,12 +42,16 @@ struct CapturedProcess { } impl CapturedProcess { - fn start(program: &'static str, command: &mut Command) -> Result { + fn start( + program: &'static str, + command: &mut Command, + spawn: impl FnOnce(&mut Command) -> Result, + ) -> Result { command .stdout(Stdio::piped()) .stderr(Stdio::piped()) .process_group(0); - let mut child = command.spawn().map_err(|source| ProcessError::Io { + let mut child = spawn(command).map_err(|source| ProcessError::Io { program, action: "spawn", source, diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index d0308a4..664949f 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -23,14 +23,20 @@ pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { source, } })?; + let process_directory = repository_root.process_directory().map_err(|source| { + DocumentationError::RepositoryRootInspect { + path: repository_path.to_owned(), + source, + } + })?; verify_root(&repository_root, repository_path)?; contributor_contract::check(&repository_root)?; node_toolchain::check(&repository_root)?; - dependabot::check(repository_path, &repository_root)?; + dependabot::check(&repository_root, &process_directory)?; workflow_contract::check(&repository_root)?; - let markdown = corpus::SourceCorpus::markdown(repository_path)?; - let workflows = corpus::SourceCorpus::workflow(repository_path)?; - execution::run(repository_path, markdown.paths(), workflows.paths())?; + let markdown = corpus::SourceCorpus::markdown(&repository_root, &process_directory)?; + let workflows = corpus::SourceCorpus::workflow(&repository_root, &process_directory)?; + execution::run(&process_directory, markdown.paths(), workflows.paths())?; verify_root(&repository_root, repository_path) } diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index 6a89664..80eafcf 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -1,13 +1,12 @@ //! This module owns deterministic documentation source selection. -use std::fs; use std::io; -use std::path::Path; use xtask::protocol_admission::posix_relative_path; use super::error::DocumentationError; -use crate::git_inventory::{GitPath, paths}; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryProcessDirectory, RepositoryRoot}; const MARKDOWN_PRESENT: [&str; 7] = [ "ls-files", @@ -49,28 +48,38 @@ enum CorpusKind { } impl SourceCorpus { - pub(super) fn markdown(repository_root: &Path) -> Result { - Self::read(repository_root, CorpusKind::Markdown) + pub(super) fn markdown( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + Self::read(repository_root, process_directory, CorpusKind::Markdown) } - pub(super) fn workflow(repository_root: &Path) -> Result { - Self::read(repository_root, CorpusKind::Workflow) + pub(super) fn workflow( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + Self::read(repository_root, process_directory, CorpusKind::Workflow) } pub(super) fn paths(&self) -> &[String] { &self.paths } - fn read(repository_root: &Path, kind: CorpusKind) -> Result { - let present = paths( - repository_root, + fn read( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + kind: CorpusKind, + ) -> Result { + let present = paths_with( kind.present_arguments(), kind.present_operation(), + |command| process_directory.spawn(command), )?; - let deleted = paths( - repository_root, + let deleted = paths_with( kind.deleted_arguments(), kind.deleted_operation(), + |command| process_directory.spawn(command), )?; let selected = present.difference(&deleted); let paths = admit_paths(repository_root, selected, kind)?; @@ -120,7 +129,7 @@ impl CorpusKind { } fn admit_paths<'a>( - repository_root: &Path, + repository_root: &RepositoryRoot, paths: impl Iterator, kind: CorpusKind, ) -> Result, DocumentationError> { @@ -134,7 +143,7 @@ fn admit_paths<'a>( } fn admit_path( - repository_root: &Path, + repository_root: &RepositoryRoot, path: &GitPath, kind: CorpusKind, ) -> Result, DocumentationError> { @@ -148,24 +157,28 @@ fn admit_path( corpus: kind.label(), path: text.clone(), })?; - let metadata = match fs::symlink_metadata(repository_root.join(relative)) { - Ok(metadata) => metadata, - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(source) => { - return Err(DocumentationError::Inspect { + match repository_root.open_file(&relative) { + Ok(_file) => Ok(Some(text)), + Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { + Ok(None) + } + Err(OpenRepositoryFileError::Io(source)) + if source.raw_os_error() == Some(rustix::io::Errno::LOOP.raw_os_error()) => + { + Err(DocumentationError::NonRegular { corpus: kind.label(), path: text, - source, - }); + }) } - }; - if metadata.file_type().is_file() { - Ok(Some(text)) - } else { - Err(DocumentationError::NonRegular { + Err(OpenRepositoryFileError::Io(source)) => Err(DocumentationError::Inspect { corpus: kind.label(), path: text, - }) + source, + }), + Err(OpenRepositoryFileError::NonRegular) => Err(DocumentationError::NonRegular { + corpus: kind.label(), + path: text, + }), } } diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs index b606c0c..b1e757c 100644 --- a/xtask/src/documentation_integrity/corpus/tests.rs +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -7,6 +7,7 @@ use std::process::Command; use super::{CorpusKind, SourceCorpus, admit_path}; use crate::documentation_integrity::error::DocumentationError; use crate::git_inventory::GitPath; +use crate::repository_file::RepositoryRoot; use crate::test_directory::TestDirectory; #[test] @@ -23,7 +24,9 @@ fn markdown_corpus_is_the_sorted_present_repository_set() -> Result<(), Box Result<(), Box Result<(), Box> { + let directory = TestDirectory::create("documentation-root-replacement")?; + let root = directory.path().join("repository"); + let retained = directory.path().join("retained"); + fs::create_dir(&root)?; + run_git(&root, &["init", "--quiet"])?; + write(&root, "original.md", "# Original\n")?; + let repository_root = RepositoryRoot::open(&root)?; + let process_directory = repository_root.process_directory()?; + let original_process_directory = std::env::current_dir()?; + + fs::rename(&root, &retained)?; + fs::create_dir(&root)?; + run_git(&root, &["init", "--quiet"])?; + write(&root, "substitute.md", "# Substitute\n")?; + + let corpus = SourceCorpus::markdown(&repository_root, &process_directory); + fs::remove_dir_all(&root)?; + fs::rename(&retained, &root)?; + let corpus = corpus?; + + assert_eq!(corpus.paths(), ["original.md"]); + assert_eq!(std::env::current_dir()?, original_process_directory); + drop(repository_root); + directory.close()?; + Ok(()) +} + #[cfg(unix)] #[test] fn symlinked_markdown_is_refused() -> Result<(), Box> { @@ -96,7 +133,9 @@ fn symlinked_markdown_is_refused() -> Result<(), Box> { write(root, "target.txt", "target\n")?; symlink("target.txt", root.join("linked.md"))?; - let result = SourceCorpus::markdown(root); + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let result = SourceCorpus::markdown(&repository_root, &process_directory); assert!(matches!( result, @@ -124,7 +163,9 @@ fn fifo_workflow_is_refused() -> Result<(), Box> { return Err("mkfifo fixture command failed".into()); } - let result = SourceCorpus::workflow(root); + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let result = SourceCorpus::workflow(&repository_root, &process_directory); assert!(matches!( result, @@ -143,7 +184,8 @@ fn non_utf8_markdown_path_is_refused() -> Result<(), Box> let root = directory.path(); let path = GitPath::new(b"bad\xff.md".to_vec()); - let result = admit_path(root, &path, CorpusKind::Markdown); + let repository_root = RepositoryRoot::open(root)?; + let result = admit_path(&repository_root, &path, CorpusKind::Markdown); assert!(matches!( result, diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index 803d2ab..4146950 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -3,9 +3,8 @@ mod manifest; use std::collections::BTreeSet; -use std::path::Path; -use crate::repository_file::RepositoryRoot; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; use super::error::DocumentationError; use super::repository_text; @@ -21,11 +20,11 @@ struct DependencyScope { } pub(super) fn check( - repository_path: &Path, repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, ) -> Result<(), DocumentationError> { let raw = repository_text::read(repository_root, DEPENDABOT_PATH)?; - let required = tracked_scopes(repository_path)?; + let required = tracked_scopes(process_directory)?; admit(&raw, &required) } diff --git a/xtask/src/documentation_integrity/dependabot/manifest.rs b/xtask/src/documentation_integrity/dependabot/manifest.rs index b559008..defb753 100644 --- a/xtask/src/documentation_integrity/dependabot/manifest.rs +++ b/xtask/src/documentation_integrity/dependabot/manifest.rs @@ -1,11 +1,10 @@ //! This module owns tracked dependency-manifest scope discovery. use std::collections::BTreeSet; -use std::path::Path; - use xtask::protocol_admission::posix_relative_path; -use crate::git_inventory::{GitPath, paths}; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::RepositoryProcessDirectory; use super::DependencyScope; use crate::documentation_integrity::error::DocumentationError; @@ -19,12 +18,12 @@ const MANIFEST_ARGUMENTS: [&str; 5] = [ ]; pub(super) fn tracked_scopes( - repository_root: &Path, + process_directory: &RepositoryProcessDirectory, ) -> Result, DocumentationError> { - paths( - repository_root, + paths_with( &MANIFEST_ARGUMENTS, "list tracked dependency manifests", + |command| process_directory.spawn(command), )? .iter() .map(manifest_scope) diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs index 4de7395..ce56642 100644 --- a/xtask/src/documentation_integrity/dependabot/tests.rs +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -106,7 +106,8 @@ fn committed_dependabot_policy_covers_every_tracked_manifest() .parent() .ok_or("xtask manifest has no repository parent")?; let repository_root = RepositoryRoot::open(root)?; - super::check(root, &repository_root)?; + let process_directory = repository_root.process_directory()?; + super::check(&repository_root, &process_directory)?; assert!(repository_root.is_current_path()?); Ok(()) } diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index b53841a..a123eb9 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -1,10 +1,10 @@ //! This module owns bounded execution of admitted documentation tools. -use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; use crate::bounded_process::{self, ProcessOutput}; +use crate::repository_file::RepositoryProcessDirectory; use super::error::DocumentationError; use super::tool::DocumentationTool; @@ -20,16 +20,16 @@ trait ToolRunner { } struct ExternalToolRunner<'a> { - repository_root: &'a Path, + process_directory: &'a RepositoryProcessDirectory, } pub(super) fn run( - repository_root: &Path, + process_directory: &RepositoryProcessDirectory, markdown: &[String], workflows: &[String], ) -> Result<(), DocumentationError> { run_with( - &mut ExternalToolRunner { repository_root }, + &mut ExternalToolRunner { process_directory }, markdown, workflows, ) @@ -143,23 +143,24 @@ impl ToolRunner for ExternalToolRunner<'_> { arguments: &[String], ) -> Result { let mut command = Command::new(tool.program()); - command - .args(arguments) - .current_dir(self.repository_root) - .stdin(Stdio::null()); - bounded_process::capture(tool.program(), &mut command, Some(TOOL_DEADLINE)).map_err( - |source| { - if source.is_not_found() { - DocumentationError::ToolUnavailable { - program: tool.program(), - install_version: tool.install_version(), - source, - } - } else { - DocumentationError::Process(source) - } - }, + command.args(arguments).stdin(Stdio::null()); + bounded_process::capture_with( + tool.program(), + &mut command, + Some(TOOL_DEADLINE), + |command| self.process_directory.spawn(command), ) + .map_err(|source| { + if source.is_not_found() { + DocumentationError::ToolUnavailable { + program: tool.program(), + install_version: tool.install_version(), + source, + } + } else { + DocumentationError::Process(source) + } + }) } } diff --git a/xtask/src/documentation_integrity/execution/external_tests.rs b/xtask/src/documentation_integrity/execution/external_tests.rs index b581f2a..cc8708f 100644 --- a/xtask/src/documentation_integrity/execution/external_tests.rs +++ b/xtask/src/documentation_integrity/execution/external_tests.rs @@ -2,6 +2,7 @@ use std::fs; +use crate::repository_file::RepositoryRoot; use crate::test_directory::TestDirectory; use super::{DocumentationError, DocumentationTool, ExternalToolRunner}; @@ -15,9 +16,11 @@ fn broken_internal_fragment_is_refused() -> Result<(), Box Result<(), Box> { directory.path().join(".github/workflows/invalid.yml"), "name: Invalid\non: [push\n", )?; + let repository_root = RepositoryRoot::open(directory.path())?; + let process_directory = repository_root.process_directory()?; let refusal = { let mut runner = ExternalToolRunner { - repository_root: directory.path(), + process_directory: &process_directory, }; super::admit_version(&mut runner, DocumentationTool::Actionlint)?; super::run_check( diff --git a/xtask/src/git_inventory.rs b/xtask/src/git_inventory.rs index 5ca55ca..917d6f6 100644 --- a/xtask/src/git_inventory.rs +++ b/xtask/src/git_inventory.rs @@ -6,4 +6,4 @@ mod process; pub(crate) use error::{GitInventoryError, GitOutputUnit}; pub(crate) use path_stream::GitPath; -pub(crate) use process::paths; +pub(crate) use process::{paths, paths_with}; diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index e43b787..25925b3 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -30,27 +30,36 @@ pub(crate) fn paths( arguments: &[&str], operation: &'static str, ) -> Result, GitInventoryError> { - let process = start_git(repository_root, arguments, operation)?; + paths_with(arguments, operation, |command| { + command.current_dir(repository_root).spawn() + }) +} + +pub(crate) fn paths_with( + arguments: &[&str], + operation: &'static str, + spawn: impl FnOnce(&mut Command) -> Result, +) -> Result, GitInventoryError> { + let process = start_git(arguments, operation, spawn)?; let paths = read_paths(process.stdout, operation); collect_git_result(process.child, process.diagnostic_worker, paths, operation) } fn start_git( - repository_root: &Path, arguments: &[&str], operation: &'static str, + spawn: impl FnOnce(&mut Command) -> Result, ) -> Result { - let mut child = Command::new("git") + let mut command = Command::new("git"); + command .args(arguments) - .current_dir(repository_root) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|source| GitInventoryError::Run { - operation, - action: "start", - source, - })?; + .stderr(Stdio::piped()); + let mut child = spawn(&mut command).map_err(|source| GitInventoryError::Run { + operation, + action: "start", + source, + })?; let Some(stdout) = child.stdout.take() else { let primary = GitInventoryError::Pipe { operation, diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs index b069669..61ceb62 100644 --- a/xtask/src/repository_file.rs +++ b/xtask/src/repository_file.rs @@ -7,11 +7,14 @@ use std::fs::File; use std::io; +use std::os::fd::OwnedFd; use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; use cap_std::ambient_authority; use cap_std::fs::{Dir, OpenOptions}; +use repository_process_spawn::set_working_directory; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ReadAccessPolicy { @@ -40,6 +43,14 @@ pub(crate) struct RepositoryRoot { path: PathBuf, } +/// An exact child-process working directory backed by an owned descriptor. +/// +/// Child setup changes directory through this descriptor, never by reopening +/// the ambient repository path. +pub(crate) struct RepositoryProcessDirectory { + directory: OwnedFd, +} + #[derive(Eq, PartialEq)] struct DirectoryIdentity { device: u64, @@ -72,6 +83,12 @@ impl RepositoryRoot { Ok(self.identity == identity) } + /// Returns an exact child-process handle for this opened directory. + pub(crate) fn process_directory(&self) -> Result { + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, 0)?; + Ok(RepositoryProcessDirectory { directory }) + } + pub(crate) fn open_file(&self, relative: &Path) -> Result { let file = self .directory @@ -87,6 +104,18 @@ impl RepositoryRoot { } } +impl RepositoryProcessDirectory { + /// Starts a child from the exact opened repository directory. + /// + /// The child changes directory through its retained descriptor after fork + /// and before exec. Parent process state is never changed. + pub(crate) fn spawn(&self, command: &mut Command) -> Result { + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, 0)?; + set_working_directory(command, directory); + command.spawn() + } +} + impl RepositoryReadPolicy { #[cfg(test)] pub(super) const fn read_access(self) -> ReadAccessPolicy { From 33b7982ac1ec62b267d12be4938fababb81d7338 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:34:40 -0700 Subject: [PATCH 041/113] Fix: guard child groups from terminal interrupts --- CHANGELOG.md | 4 +- Cargo.lock | 21 ++ ...007-terminal-signal-process-group-guard.md | 71 ++++++ docs/adr/README.md | 1 + .../cap-std-and-cap-fs-ext-4.0.2.md | 4 + docs/dependencies/signal-hook-0.4.4.md | 81 +++++++ xtask/Cargo.toml | 3 + xtask/src/bounded_process.rs | 5 +- xtask/src/bounded_process/capture.rs | 54 +++-- xtask/src/bounded_process/error.rs | 14 +- xtask/src/bounded_process/interrupt.rs | 212 ++++++++++++++++++ xtask/src/bounded_process/interrupt/tests.rs | 34 +++ xtask/src/bounded_process/process_group.rs | 2 + .../process_group/child_tests.rs | 43 +++- .../bounded_process/process_group/tests.rs | 58 ++++- xtask/src/bounded_process/reader.rs | 61 ++--- 16 files changed, 608 insertions(+), 60 deletions(-) create mode 100644 docs/adr/0007-terminal-signal-process-group-guard.md create mode 100644 docs/dependencies/signal-hook-0.4.4.md create mode 100644 xtask/src/bounded_process/interrupt.rs create mode 100644 xtask/src/bounded_process/interrupt/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 959e732..e2661b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,9 @@ after its public API and format compatibility policies are established. preserve non-UTF-8 template paths without lossy conversion. Documentation Git inventory and tools start from one retained repository directory handle, so transient replacement of the ambient repository path cannot redirect - validation. + validation. Terminal signals now become typed refusals while an external + repository task is active, so captured and inherited child groups are killed + and reaped before `xtask` returns. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/Cargo.lock b/Cargo.lock index fbded8d..5775a7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -479,6 +479,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "syn" version = "2.0.119" @@ -719,6 +739,7 @@ dependencies = [ "rustix", "serde", "serde_json", + "signal-hook", "yaml-rust2", ] diff --git a/docs/adr/0007-terminal-signal-process-group-guard.md b/docs/adr/0007-terminal-signal-process-group-guard.md new file mode 100644 index 0000000..9d52445 --- /dev/null +++ b/docs/adr/0007-terminal-signal-process-group-guard.md @@ -0,0 +1,71 @@ +# ADR-0007: Terminal Signal Process-Group Guard + +- Status: Accepted +- Date: 2026-07-28 +- Owners: Keep repository verification +- Related issue: none — review remediation for pull request 61 +- Depends on: ADR-0006 + +## Context + +Repository tasks run external tools in dedicated process groups. A deadline or +collection failure terminates the whole group, but the operating system's +default terminal-signal action terminates the `xtask` parent immediately. +Signals such as `SIGINT` therefore bypass Rust cleanup and can leave an +isolated child or descendant running. + +Captured output creates a second wait boundary. A child leader can exit while +a descendant retains an inherited pipe, so guarding only the child wait does +not cover the complete operation. + +## Decision + +While an external repository task is active, `xtask` installs one process-wide +guard for `SIGINT`, `SIGTERM`, `SIGHUP`, and `SIGQUIT`. A dedicated signal +thread records the first signal for every active operation. Child waits and +captured-output readers poll that state at the existing ten-millisecond process +interval. + +An observed terminal signal becomes a typed `ProcessError::Interrupted` +refusal. The normal failure path then sends `SIGKILL` to the dedicated child +process group, kills and reaps the child, and joins captured-output readers +before returning. The terminal signal is not sent directly to the child group; +one cleanup authority avoids races between signal delivery and mandatory +process-group termination. + +When no external operation is active, the guard restores the signal's default +behavior. A signal observed while the final operation retires, or a second +signal received during cleanup, also restores default termination instead of +being swallowed. Handler registration is initialized once because removing a +`signal-hook` action does not restore a previous operating-system handler. The +[signal-hook dependency admission](../dependencies/signal-hook-0.4.4.md) +records the selected package boundary. + +## Alternatives considered + +- Relying on the default terminal action abandons Rust cleanup and can strand + descendants. +- Forwarding the original terminal signal directly to the child group races + mandatory cleanup. On macOS, a descendant can exit from `SIGINT` before the + subsequent group kill, which makes the second operation report `EPERM` + despite successful termination. +- A parent-death signal is not portable across the supported Unix platforms + and does not cover descendants that change their parent relationship. +- Polling only the child status misses descendants that retain captured output + pipes after the child leader exits. + +## Consequences + +Terminal interruption follows the same typed cleanup and reaping protocol as +timeouts and collection failures. Unbounded child waits and output collection +now wake at most one process-poll interval after the signal thread records an +interrupt. + +Signal registration is process-global, but active state is operation-local and +supports concurrent repository tasks. No lock is held while a child is +spawned, polled, killed, reaped, or read. + +The guard is private to `xtask`. It changes no Keep library API, content +identity, durable format, or recovery protocol. Regression evidence sends +`SIGINT` to a supervisor with an isolated descendant, requires the exact typed +refusal, and proves the descendant is no longer reachable. diff --git a/docs/adr/README.md b/docs/adr/README.md index 622275a..0521dfb 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -58,3 +58,4 @@ encryption, concurrency, or public-API surface it governs. - [ADR-0004: Hexagonal boundary architecture](0004-hexagonal-boundary-architecture.md) - [ADR-0005: Durable segment store protocol](0005-durable-segment-store-protocol.md) - [ADR-0006: Descriptor-bound child working directory](0006-descriptor-bound-child-working-directory.md) +- [ADR-0007: Terminal signal process-group guard](0007-terminal-signal-process-group-guard.md) diff --git a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md index 24a3ab3..ffc681b 100644 --- a/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md +++ b/docs/dependencies/cap-std-and-cap-fs-ext-4.0.2.md @@ -29,6 +29,10 @@ The bounded subprocess adapter uses Rustix's safe process API to send `SIGKILL` to a dedicated child process group after a subprocess deadline or collection failure. This prevents descendants that inherited an output pipe from surviving the failed repository task. +The +[signal-hook dependency admission](signal-hook-0.4.4.md) +records the terminal-signal guard that routes interruption through the same +authoritative cleanup boundary. Documentation verification also duplicates the admitted repository directory handle and uses the isolated `repository-process-spawn` crate to start Git and diff --git a/docs/dependencies/signal-hook-0.4.4.md b/docs/dependencies/signal-hook-0.4.4.md new file mode 100644 index 0000000..68d1e77 --- /dev/null +++ b/docs/dependencies/signal-hook-0.4.4.md @@ -0,0 +1,81 @@ +# Dependency Admission: signal-hook 0.4.4 + +- Status: Accepted for repository-task terminal-signal handling only +- Date: 2026-07-28 +- Owner: Keep repository verification +- Upstream: + [rust-cli/signal-hook](https://github.com/rust-cli/signal-hook) + +## Admitted use + +Keep admits the exactly pinned `signal-hook` 0.4.4 package only behind the +`xtask` crate's `repository-tasks` feature. The private bounded-process adapter +uses its iterator API to observe `SIGINT`, `SIGTERM`, `SIGHUP`, and `SIGQUIT` +on a dedicated thread while external repository tools are active. + +The adapter records the first signal as a typed interruption. Existing +process-group cleanup then terminates and reaps the child and its descendants. +When no repository tool is active, the adapter invokes the operating system's +default signal behavior. The +[terminal signal process-group decision](../adr/0007-terminal-signal-process-group-guard.md) +records the process-wide concurrency and cleanup policy. + +The package is absent from Keep's published library graph, public API, content +identities, durable formats, and production storage behavior. No +dependency-owned type crosses out of the private repository-task adapter. + +## Why the standard library is insufficient + +The standard library exposes Unix signal constants through platform APIs but +does not provide a safe registration and delivery mechanism for process +signals. Implementing one locally would require unsafe signal-handler code, +async-signal-safety analysis, self-pipe or equivalent wakeup machinery, handler +composition, and platform-specific restoration behavior. + +The admitted package owns that unsafe operating-system boundary. Keep-owned +code consumes signals only through its safe iterator and default-handler APIs. + +## Features and resolved graph + +The direct dependency disables default features and enables only `iterator`. +That feature also enables `channel`. Both are activated solely by +`repository-tasks`. + +The locked graph introduced for this boundary is: + +- `signal-hook` 0.4.4; +- `signal-hook-registry` 1.4.8; +- `errno` 0.3.14; and +- `libc` 0.2.186. + +The latter two packages were already present in the locked workspace graph. + +## Safety, licensing, and compatibility + +`signal-hook` 0.4.4 and `signal-hook-registry` 1.4.8 declare +`MIT OR Apache-2.0`. Their declared Rust-version floors are 1.66 and 1.26, +respectively. Compatibility remains established by Keep's pinned stable, MSRV, +debug, release, Clippy, dependency-policy, and advisory lanes. + +The packages contain unsafe code around operating-system signal registration +and delivery. Keep-owned code invokes only safe APIs, performs no work in a +signal handler, retains no dependency-owned public type, and confines global +registration to the private repository-task adapter. `cargo deny` and RustSec +checks remain mandatory point-in-time evidence. + +## Failure and recovery boundaries + +Registration, iterator creation, or signal-thread creation failure is a typed +refusal before the external command starts. Registry poisoning also refuses +the task. Once a signal is observed, process-group cleanup remains authoritative +and any cleanup failure is preserved alongside the interruption. + +No repository, durable, or authoritative application state is written by this +boundary. Keep can remove the dependency without changing public or durable +behavior by replacing it with an equally portable safe mechanism that +preserves default behavior outside active tasks, typed interruption, bounded +wakeup, concurrent-operation handling, and whole-process-group cleanup. + +Reopen this admission if the direct version, selected features, resolved graph, +license, supported signals, process-global handler policy, or +repository-task-only boundary changes. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 854dec0..7f636da 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -18,6 +18,7 @@ repository-tasks = [ "dep:rustix", "dep:serde", "dep:serde_json", + "dep:signal-hook", "dep:yaml-rust2", ] @@ -37,6 +38,8 @@ rustix = { version = "=1.1.4", default-features = false, features = ["process", serde = { version = "=1.0.229", default-features = false, features = ["std"], optional = true } # Typed JSON admission checks the committed documentation-tool lock graph. serde_json = { version = "=1.0.151", default-features = false, features = ["std"], optional = true } +# Safe self-pipe delivery forwards terminal signals to active child groups. +signal-hook = { version = "=0.4.4", default-features = false, features = ["iterator"], optional = true } # Pure Rust YAML admission identifies executable GitHub Actions steps. yaml-rust2 = { version = "=0.11.0", default-features = false, optional = true } diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index a2f0240..2dfb7bd 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -4,6 +4,7 @@ mod capture; mod cleanup; mod deadline; mod error; +mod interrupt; mod process_group; mod reader; @@ -15,6 +16,7 @@ use capture::wait_for_child; pub(crate) use capture::{capture, capture_with}; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; +use interrupt::InterruptGuard; use reader::ReaderWorker; /// The completed child status and any output retained by the selected mode. @@ -45,13 +47,14 @@ pub(crate) fn status( deadline: Option, ) -> Result { let deadline = ProcessDeadline::new(program, deadline)?; + let interrupts = InterruptGuard::begin(program)?; command.process_group(0); let mut child = command.spawn().map_err(|source| ProcessError::Io { program, action: "spawn", source, })?; - let status = wait_for_child(program, &mut child, &deadline)?; + let status = wait_for_child(program, &mut child, &deadline, &interrupts)?; Ok(ProcessOutput { code: status.code(), succeeded: status.success(), diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 56d19f7..0a47a4b 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -3,13 +3,14 @@ use std::os::unix::process::CommandExt; use std::process::{Child, Command, ExitStatus, Stdio}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::Duration; use super::cleanup::{cleanup_process, join_after_cleanup, join_readers}; -use super::{ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker}; +use super::{InterruptGuard, ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker}; use crate::process_output::BoundedBytes; const OUTPUT_LIMIT: usize = 1_048_576; +const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(10); /// Runs a child synchronously and captures bounded standard output and error. /// @@ -32,11 +33,13 @@ pub(crate) fn capture_with( spawn: impl FnOnce(&mut Command) -> Result, ) -> Result { let deadline = ProcessDeadline::new(program, deadline)?; - CapturedProcess::start(program, command, spawn)?.finish(program, &deadline) + let interrupts = InterruptGuard::begin(program)?; + CapturedProcess::start(program, command, spawn, interrupts)?.finish(program, &deadline) } struct CapturedProcess { child: Child, + interrupts: InterruptGuard, stderr: ReaderWorker, stdout: ReaderWorker, } @@ -46,6 +49,7 @@ impl CapturedProcess { program: &'static str, command: &mut Command, spawn: impl FnOnce(&mut Command) -> Result, + interrupts: InterruptGuard, ) -> Result { command .stdout(Stdio::piped()) @@ -83,6 +87,7 @@ impl CapturedProcess { }; Ok(Self { child, + interrupts, stderr, stdout, }) @@ -93,15 +98,15 @@ impl CapturedProcess { program: &'static str, deadline: &ProcessDeadline, ) -> Result { - let status = match wait_for_child(program, &mut self.child, deadline) { + let status = match wait_for_child(program, &mut self.child, deadline, &self.interrupts) { Ok(status) => status, Err(error) => return Err(join_readers(self.stdout, self.stderr, error)), }; - let stdout = match self.stdout.receive(deadline) { + let stdout = match self.stdout.receive(deadline, &self.interrupts) { Ok(output) => output, Err(error) => return Err(self.cleanup_readers(error)), }; - let stderr = match self.stderr.receive(deadline) { + let stderr = match self.stderr.receive(deadline, &self.interrupts) { Ok(output) => output, Err(error) => return Err(self.cleanup_readers(error)), }; @@ -115,6 +120,9 @@ impl CapturedProcess { refuse_exceeded(program, "stdout", &stdout) .and_then(|()| refuse_exceeded(program, "stderr", &stderr)) .map_err(|error| cleanup_process(&mut self.child, error))?; + if let Some(error) = self.interrupts.refusal(program) { + return Err(cleanup_process(&mut self.child, error)); + } Ok(ProcessOutput { code: status.code(), succeeded: status.success(), @@ -134,20 +142,12 @@ pub(super) fn wait_for_child( program: &'static str, child: &mut Child, deadline: &ProcessDeadline, + interrupts: &InterruptGuard, ) -> Result { - let ProcessDeadline::Bounded { duration, expires } = deadline else { - return child.wait().map_err(|source| { - cleanup_process( - child, - ProcessError::Io { - program, - action: "wait", - source, - }, - ) - }); - }; loop { + if let Some(error) = interrupts.refusal(program) { + return Err(cleanup_process(child, error)); + } match child.try_wait() { Err(source) => { return Err(cleanup_process( @@ -160,18 +160,14 @@ pub(super) fn wait_for_child( )); } Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() >= *expires => { - return Err(cleanup_process( - child, - ProcessError::Timeout { - program, - duration: *duration, - }, - )); + Ok(None) => { + let interval = match deadline.remaining(program) { + Ok(Some((remaining, _duration))) => PROCESS_POLL_INTERVAL.min(remaining), + Ok(None) => PROCESS_POLL_INTERVAL, + Err(error) => return Err(cleanup_process(child, error)), + }; + thread::sleep(interval); } - Ok(None) => thread::sleep( - Duration::from_millis(10).min(expires.saturating_duration_since(Instant::now())), - ), } } } diff --git a/xtask/src/bounded_process/error.rs b/xtask/src/bounded_process/error.rs index 5a35331..cc46e8e 100644 --- a/xtask/src/bounded_process/error.rs +++ b/xtask/src/bounded_process/error.rs @@ -24,6 +24,11 @@ pub(crate) enum ProcessError { action: &'static str, source: io::Error, }, + /// A terminal signal interrupted the complete child operation. + Interrupted { + program: &'static str, + signal: &'static str, + }, /// A child configured for capture did not expose the requested pipe. MissingStream { program: &'static str, @@ -55,7 +60,8 @@ impl ProcessError { primary.is_not_found() } Self::Io { source, .. } => source.kind() == io::ErrorKind::NotFound, - Self::MissingStream { .. } + Self::Interrupted { .. } + | Self::MissingStream { .. } | Self::OutputLimit { .. } | Self::ReaderPanic { .. } | Self::Timeout { .. } => false, @@ -82,6 +88,9 @@ impl fmt::Display for ProcessError { Self::Io { program, action, .. } => write!(formatter, "cannot {action} {program} process"), + Self::Interrupted { program, signal } => { + write!(formatter, "{program} process was interrupted by {signal}") + } Self::MissingStream { program, stream } => { write!(formatter, "{program} {stream} pipe is unavailable") } @@ -110,7 +119,8 @@ impl Error for ProcessError { match self { Self::Additional { primary, .. } => Some(primary), Self::Cleanup { source, .. } | Self::Io { source, .. } => Some(source), - Self::MissingStream { .. } + Self::Interrupted { .. } + | Self::MissingStream { .. } | Self::OutputLimit { .. } | Self::ReaderPanic { .. } | Self::Timeout { .. } => None, diff --git a/xtask/src/bounded_process/interrupt.rs b/xtask/src/bounded_process/interrupt.rs new file mode 100644 index 0000000..fd4f978 --- /dev/null +++ b/xtask/src/bounded_process/interrupt.rs @@ -0,0 +1,212 @@ +//! This module owns terminal-signal refusal for active child groups. + +use std::io; +use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::thread; + +use signal_hook::consts::{SIGHUP, SIGINT, SIGQUIT, SIGTERM}; +use signal_hook::iterator::Signals; + +use super::ProcessError; + +const HANDLED_SIGNALS: [i32; 4] = [SIGINT, SIGTERM, SIGHUP, SIGQUIT]; + +static CONTROLLER: OnceLock = OnceLock::new(); +static CONTROLLER_START: Mutex<()> = Mutex::new(()); + +pub(super) struct InterruptGuard { + registry: Arc>>>, + state: Arc, +} + +struct InterruptController { + registry: Arc>>>, +} + +struct InterruptState { + observed: AtomicBool, + signal: AtomicI32, +} + +enum SignalAdmission { + First, + Repeated, +} + +enum DispatchOutcome { + Admitted, + UseDefault, +} + +impl InterruptGuard { + pub(super) fn begin(program: &'static str) -> Result { + interrupt_controller(program)?.register(program) + } + + pub(super) fn refusal(&self, program: &'static str) -> Option { + let signal = self.state.observe()?; + let signal_name = signal_hook::low_level::signal_name(signal).unwrap_or("unknown signal"); + Some(ProcessError::Interrupted { + program, + signal: signal_name, + }) + } +} + +fn interrupt_controller( + program: &'static str, +) -> Result<&'static InterruptController, ProcessError> { + if let Some(controller) = CONTROLLER.get() { + return Ok(controller); + } + let start_guard = CONTROLLER_START.lock().map_err(|_| ProcessError::Io { + program, + action: "serialize terminal signal guard initialization for", + source: io::Error::other("terminal signal initializer is poisoned"), + })?; + if CONTROLLER.get().is_none() { + let controller = InterruptController::start().map_err(|source| ProcessError::Io { + program, + action: "initialize terminal signal guard for", + source, + })?; + CONTROLLER.set(controller).map_err(|_| ProcessError::Io { + program, + action: "publish terminal signal guard for", + source: io::Error::new( + io::ErrorKind::AlreadyExists, + "terminal signal controller was already initialized", + ), + })?; + } + let controller = CONTROLLER.get().ok_or_else(|| ProcessError::Io { + program, + action: "read terminal signal guard for", + source: io::Error::other("terminal signal controller was not initialized"), + })?; + drop(start_guard); + Ok(controller) +} + +impl Drop for InterruptGuard { + fn drop(&mut self) { + if let Ok(mut registry) = self.registry.lock() { + registry.retain(|candidate| { + candidate + .upgrade() + .is_some_and(|state| !Arc::ptr_eq(&state, &self.state)) + }); + } + if let Some(signal) = self.state.unobserved() { + terminate_by_default(signal); + } + } +} + +impl InterruptController { + fn start() -> Result { + let signals = Signals::new(HANDLED_SIGNALS)?; + let registry = Arc::new(Mutex::new(Vec::new())); + let signal_registry = Arc::clone(®istry); + let worker = thread::Builder::new() + .name(String::from("xtask-signal-guard")) + .spawn(move || dispatch(signals, &signal_registry))?; + drop(worker); + Ok(Self { registry }) + } + + fn register(&self, program: &'static str) -> Result { + let state = Arc::new(InterruptState::new()); + self.registry + .lock() + .map_err(|_| ProcessError::Io { + program, + action: "register active child for", + source: io::Error::other("terminal signal registry is poisoned"), + })? + .push(Arc::downgrade(&state)); + Ok(InterruptGuard { + registry: Arc::clone(&self.registry), + state, + }) + } +} + +impl InterruptState { + const fn new() -> Self { + Self { + observed: AtomicBool::new(false), + signal: AtomicI32::new(0), + } + } + + fn interrupt(&self, signal: i32) -> SignalAdmission { + if self + .signal + .compare_exchange(0, signal, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + SignalAdmission::First + } else { + SignalAdmission::Repeated + } + } + + fn observe(&self) -> Option { + let signal = self.signal.load(Ordering::Acquire); + if signal == 0 { + return None; + } + self.observed.store(true, Ordering::Release); + Some(signal) + } + + fn unobserved(&self) -> Option { + let signal = self.signal.load(Ordering::Acquire); + (signal != 0 && !self.observed.load(Ordering::Acquire)).then_some(signal) + } +} + +fn dispatch(mut signals: Signals, registry: &Mutex>>) { + for signal in signals.forever() { + if !matches!( + dispatch_signal(registry, signal), + Ok(DispatchOutcome::Admitted) + ) { + terminate_by_default(signal); + } + } +} + +fn dispatch_signal( + registry: &Mutex>>, + signal: i32, +) -> Result { + let mut registry = registry.lock().map_err(|_| ())?; + let mut has_active_operation = false; + let mut repeated = false; + registry.retain(|candidate| { + candidate.upgrade().is_some_and(|state| { + has_active_operation = true; + repeated |= matches!(state.interrupt(signal), SignalAdmission::Repeated); + true + }) + }); + drop(registry); + if !has_active_operation || repeated { + Ok(DispatchOutcome::UseDefault) + } else { + Ok(DispatchOutcome::Admitted) + } +} + +fn terminate_by_default(signal: i32) { + if signal_hook::low_level::emulate_default_handler(signal).is_err() { + signal_hook::low_level::abort(); + } +} + +#[cfg(test)] +#[path = "interrupt/tests.rs"] +mod tests; diff --git a/xtask/src/bounded_process/interrupt/tests.rs b/xtask/src/bounded_process/interrupt/tests.rs new file mode 100644 index 0000000..a97ee35 --- /dev/null +++ b/xtask/src/bounded_process/interrupt/tests.rs @@ -0,0 +1,34 @@ +//! This module owns terminal-signal guard state-transition laws. + +use signal_hook::consts::{SIGINT, SIGTERM}; + +use super::{InterruptState, SignalAdmission}; + +#[test] +fn first_terminal_signal_becomes_one_observed_refusal() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGINT), SignalAdmission::First)); + assert_eq!(state.observe(), Some(SIGINT)); + assert_eq!(state.unobserved(), None); +} + +#[test] +fn retirement_detects_an_unobserved_terminal_signal() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGTERM), SignalAdmission::First)); + assert_eq!(state.unobserved(), Some(SIGTERM)); +} + +#[test] +fn a_second_terminal_signal_requires_default_termination() { + let state = InterruptState::new(); + + assert!(matches!(state.interrupt(SIGINT), SignalAdmission::First)); + assert!(matches!( + state.interrupt(SIGTERM), + SignalAdmission::Repeated + )); + assert_eq!(state.observe(), Some(SIGINT)); +} diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs index 50100d1..558033b 100644 --- a/xtask/src/bounded_process/process_group.rs +++ b/xtask/src/bounded_process/process_group.rs @@ -33,6 +33,8 @@ const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; const DESCENDANT_READY: &str = "KEEP_XTASK_DESCENDANT_READY"; #[cfg(test)] const DESCENDANT_SOCKET: &str = "KEEP_XTASK_DESCENDANT_SOCKET"; +#[cfg(test)] +const INTERRUPT_SUPERVISOR: &str = "KEEP_XTASK_INTERRUPT_SUPERVISOR"; #[cfg(test)] fn wait_for_ready(path: &std::path::Path) -> Result<(), io::Error> { diff --git a/xtask/src/bounded_process/process_group/child_tests.rs b/xtask/src/bounded_process/process_group/child_tests.rs index 2a234d0..51327d2 100644 --- a/xtask/src/bounded_process/process_group/child_tests.rs +++ b/xtask/src/bounded_process/process_group/child_tests.rs @@ -8,8 +8,49 @@ use std::path::Path; use std::process::Command; use super::{ - DESCENDANT_CHILD, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, wait_for_ready, + DESCENDANT_CHILD, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, + wait_for_ready, }; +use crate::bounded_process::{ProcessError, capture}; + +#[test] +fn process_supervisor_captures_descendant_until_interrupted() -> Result<(), io::Error> { + if env::var_os(INTERRUPT_SUPERVISOR).is_none() { + return Ok(()); + } + let executable = env::current_exe()?; + let mut command = Command::new(executable); + command + .args([ + "--exact", + "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open", + ]) + .env(DESCENDANT_PARENT, "1") + .env( + DESCENDANT_READY, + env::var_os(DESCENDANT_READY).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") + })?, + ) + .env( + DESCENDANT_SOCKET, + env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing descendant socket path") + })?, + ); + match capture("interrupt fixture", &mut command, None) { + Err(ProcessError::Interrupted { + program: "interrupt fixture", + signal: "SIGINT", + }) => Ok(()), + Err(error) => Err(io::Error::other(format!( + "unexpected interrupt refusal: {error}" + ))), + Ok(_) => Err(io::Error::other( + "interrupt fixture unexpectedly completed successfully", + )), + } +} #[test] fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 67fab14..72a4614 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -9,13 +9,53 @@ use std::process::{Command, Stdio}; use std::thread; use std::time::{Duration, Instant}; -use super::{DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, wait_for_ready}; +use rustix::process::{Pid, Signal, kill_process}; + +use super::{ + DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, wait_for_ready, +}; use crate::bounded_process::cleanup::cleanup_process; use crate::bounded_process::{ProcessError, capture}; use crate::test_directory::TestDirectory; const CHILD_PROCESS: &str = "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open"; +const SUPERVISOR_PROCESS: &str = "bounded_process::process_group::child_tests::process_supervisor_captures_descendant_until_interrupted"; + +#[test] +fn terminal_interrupt_terminates_the_isolated_descendant_group() +-> Result<(), Box> { + let directory = TestDirectory::create("process-group-interrupt")?; + let ready = directory.path().join("ready"); + let socket = directory.path().join("descendant.sock"); + let executable = env::current_exe()?; + let mut supervisor = Command::new(executable) + .args(["--exact", SUPERVISOR_PROCESS]) + .env(INTERRUPT_SUPERVISOR, "1") + .env(DESCENDANT_READY, &ready) + .env(DESCENDANT_SOCKET, &socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + wait_for_ready(&ready)?; + let supervisor_pid = + Pid::from_raw(i32::try_from(supervisor.id())?).ok_or("supervisor process ID is zero")?; + kill_process(supervisor_pid, Signal::INT)?; + + let status = wait_for_exit(&mut supervisor)?; + let descendant_survived = descendant_survived_cleanup(&socket)?; + directory.close()?; + assert!( + status.success(), + "interrupted supervisor did not exit cleanly: {status:?}" + ); + assert!( + !descendant_survived, + "terminal interrupt left the isolated descendant reachable" + ); + Ok(()) +} #[test] fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> @@ -120,3 +160,19 @@ fn descendant_survived_cleanup(socket: &Path) -> Result { } } } + +fn wait_for_exit(child: &mut std::process::Child) -> Result { + let expires = Instant::now() + .checked_add(Duration::from_secs(2)) + .ok_or_else(|| io::Error::other("supervisor exit deadline overflow"))?; + loop { + if let Some(status) = child.try_wait()? { + return Ok(status); + } + if Instant::now() >= expires { + child.kill()?; + return child.wait(); + } + thread::yield_now(); + } +} diff --git a/xtask/src/bounded_process/reader.rs b/xtask/src/bounded_process/reader.rs index cecf26c..f6d937d 100644 --- a/xtask/src/bounded_process/reader.rs +++ b/xtask/src/bounded_process/reader.rs @@ -6,7 +6,9 @@ use std::thread::{self, JoinHandle}; use crate::process_output::{BoundedBytes, bounded_bytes}; -use super::{ProcessDeadline, ProcessError}; +use super::{InterruptGuard, ProcessDeadline, ProcessError}; + +const INTERRUPT_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); pub(super) struct ReaderWorker { handle: JoinHandle<()>, @@ -41,22 +43,33 @@ impl ReaderWorker { }) } - pub(super) fn receive(&self, deadline: &ProcessDeadline) -> Result { - let result = match deadline.remaining(self.program)? { - Some((remaining, duration)) => self - .receiver - .recv_timeout(remaining) - .map_err(|error| receive_error(self.program, self.stream, duration, error))?, - None => self - .receiver - .recv() - .map_err(|_| reader_panic(self.program, self.stream))?, - }; - result.map_err(|source| ProcessError::Io { - program: self.program, - action: "read child output", - source, - }) + pub(super) fn receive( + &self, + deadline: &ProcessDeadline, + interrupts: &InterruptGuard, + ) -> Result { + loop { + if let Some(error) = interrupts.refusal(self.program) { + return Err(error); + } + let (wait, duration) = receive_wait(deadline, self.program)?; + match self.receiver.recv_timeout(wait) { + Ok(result) => { + return result.map_err(|source| ProcessError::Io { + program: self.program, + action: "read child output", + source, + }); + } + Err(RecvTimeoutError::Timeout) if duration.is_none() => {} + Err(RecvTimeoutError::Timeout) => { + deadline.remaining(self.program)?; + } + Err(RecvTimeoutError::Disconnected) => { + return Err(reader_panic(self.program, self.stream)); + } + } + } } pub(super) fn join(self) -> Result<(), ProcessError> { @@ -66,15 +79,13 @@ impl ReaderWorker { } } -const fn receive_error( +fn receive_wait( + deadline: &ProcessDeadline, program: &'static str, - stream: &'static str, - duration: std::time::Duration, - error: RecvTimeoutError, -) -> ProcessError { - match error { - RecvTimeoutError::Timeout => ProcessError::Timeout { program, duration }, - RecvTimeoutError::Disconnected => reader_panic(program, stream), +) -> Result<(std::time::Duration, Option), ProcessError> { + match deadline.remaining(program)? { + Some((remaining, duration)) => Ok((remaining.min(INTERRUPT_POLL_INTERVAL), Some(duration))), + None => Ok((INTERRUPT_POLL_INTERVAL, None)), } } From 65ac90bda643134429e4837d226308313bf73d8c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:38:18 -0700 Subject: [PATCH 042/113] Fix: require every documentation CI command --- CHANGELOG.md | 5 ++-- .../workflow_contract.rs | 18 ++++-------- .../workflow_contract/tests.rs | 29 +++++++++++++++++-- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2661b1..9d73ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ after its public API and format compatibility policies are established. substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow commands as YAML, rejects guarded or non-string `run` values, preserves - declarations after Dependabot directory lists, and applies one deadline - across captured and inherited child execution and output collection. + declarations after Dependabot directory lists, requires every reviewed + documentation CI command exactly once, and applies one deadline across + captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation Git inventory and tools start from one retained repository directory handle, diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 9ec27c5..fa918a0 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -8,6 +8,8 @@ use super::error::DocumentationError; use super::repository_text; const CI_PATH: &str = ".github/workflows/ci.yml"; +const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ + documentation_integrity::execution::external_tests -- --ignored"; const XTASK_COMMAND: &str = "cargo xtask documentation-integrity-check"; const REVIEWED_RUNS: &[&str] = &[ "rustup show", @@ -16,8 +18,7 @@ scripts/install_documentation_tools.sh "$documentation_tools" printf '%s\n' \ "$documentation_tools/bin" \ "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH""#, - r"cargo test --locked --package xtask \ - documentation_integrity::execution::external_tests -- --ignored", + MALFORMED_INPUT_COMMAND, XTASK_COMMAND, r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#, ]; @@ -69,17 +70,10 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> } fn runs_are_reviewed(runs: &[String]) -> bool { - runs.iter().all(|run| REVIEWED_RUNS.contains(&run.as_str())) - && runs + runs.len() == REVIEWED_RUNS.len() + && REVIEWED_RUNS .iter() - .filter(|run| run.as_str() == "rustup show") - .count() - == 1 - && runs - .iter() - .filter(|run| run.as_str() == XTASK_COMMAND) - .count() - == 1 + .all(|required| runs.iter().filter(|run| run.as_str() == *required).count() == 1) } const fn contract(requirement: &'static str) -> DocumentationError { diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 0a4babf..91a111c 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -2,24 +2,49 @@ use std::path::Path; use crate::repository_file::RepositoryRoot; -const WORKFLOW: &str = r"name: CI +const WORKFLOW: &str = r#"name: CI jobs: documentation: name: Documentation steps: - name: Install Rust run: rustup show + - name: Install documentation tools + run: | + documentation_tools="$RUNNER_TEMP/documentation-tools" + scripts/install_documentation_tools.sh "$documentation_tools" + printf '%s\n' \ + "$documentation_tools/bin" \ + "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" + - name: Verify malformed inputs + run: | + cargo test --locked --package xtask \ + documentation_integrity::execution::external_tests -- --ignored - name: Verify run: cargo xtask documentation-integrity-check + - name: Check whitespace + run: git diff --check "$(git hash-object -t tree /dev/null)" HEAD next-job: steps: [] -"; +"#; #[test] fn documentation_job_delegates_once_to_the_rust_boundary() { assert!(super::admit(WORKFLOW).is_ok()); } +#[test] +fn documentation_job_requires_the_malformed_input_regressions() { + let runs = super::REVIEWED_RUNS + .iter() + .copied() + .filter(|run| *run != super::MALFORMED_INPUT_COMMAND) + .map(String::from) + .collect::>(); + + assert!(!super::runs_are_reviewed(&runs)); +} + #[test] fn documentation_job_refuses_python_execution() { let workflow = WORKFLOW.replace( From 27332c0fb41f366e8307d3f0a78cb80ae2f19c17 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:42:36 -0700 Subject: [PATCH 043/113] Fix: synchronize descendant deadline evidence --- .../bounded_process/process_group/tests.rs | 57 ++++++++++++++----- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 72a4614..672b2e9 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -5,7 +5,7 @@ use std::io::{self, Write}; use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Stdio}; use std::thread; use std::time::{Duration, Instant}; @@ -15,13 +15,42 @@ use super::{ DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, wait_for_ready, }; use crate::bounded_process::cleanup::cleanup_process; -use crate::bounded_process::{ProcessError, capture}; +use crate::bounded_process::{ProcessError, capture_with}; use crate::test_directory::TestDirectory; const CHILD_PROCESS: &str = "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open"; const SUPERVISOR_PROCESS: &str = "bounded_process::process_group::child_tests::process_supervisor_captures_descendant_until_interrupted"; +fn spawn_ready_descendant( + executable: &Path, + ready: &Path, + socket: &Path, +) -> Result> { + let mut child = Command::new(executable) + .args(["--exact", CHILD_PROCESS]) + .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_READY, ready) + .env(DESCENDANT_SOCKET, socket) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn()?; + if let Err(source) = wait_for_ready(ready) { + let error = cleanup_process( + &mut child, + ProcessError::Io { + program: "test process", + action: "wait for descendant readiness in", + source, + }, + ); + return Err(Box::new(error)); + } + Ok(child) +} + #[test] fn terminal_interrupt_terminates_the_isolated_descendant_group() -> Result<(), Box> { @@ -60,22 +89,22 @@ fn terminal_interrupt_terminates_the_isolated_descendant_group() #[test] fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box> { - let directory = TestDirectory::create("process-descendant-deadline")?; - let ready = directory.path().join("ready"); - let socket = directory.path().join("descendant.sock"); + let directory = TestDirectory::create("pd")?; + let ready = directory.path().join("r"); + let socket = directory.path().join("d"); let executable = env::current_exe()?; - let mut command = Command::new(executable); - command - .args(["--exact", CHILD_PROCESS]) - .env(DESCENDANT_PARENT, "1") - .env(DESCENDANT_READY, &ready) - .env(DESCENDANT_SOCKET, &socket) - .stdin(Stdio::null()); + let child = spawn_ready_descendant(&executable, &ready, &socket)?; - let result = capture( + assert!( + ready.is_file(), + "descendant must hold the output pipe before the deadline starts" + ); + let mut admitted = Command::new("pre-spawned descendant fixture"); + let result = capture_with( "test process", - &mut command, + &mut admitted, Some(Duration::from_millis(25)), + |_command| Ok(child), ); assert!(matches!( From 0b5b768e85e60c877886eb5dafe2138c1e7a5d48 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:45:27 -0700 Subject: [PATCH 044/113] Fix: parse Dependabot policy values structurally --- CHANGELOG.md | 3 +- .../src/documentation_integrity/dependabot.rs | 108 +++++++----------- .../dependabot/tests.rs | 42 ++++--- 3 files changed, 74 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d73ace..9788316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ after its public API and format compatibility policies are established. substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, parses documentation workflow commands as YAML, rejects guarded or non-string `run` values, preserves - declarations after Dependabot directory lists, requires every reviewed + declarations after Dependabot directory lists, compares Dependabot + maintenance fields as typed YAML values, requires every reviewed documentation CI command exactly once, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index 4146950..de566fd 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -4,6 +4,8 @@ mod manifest; use std::collections::BTreeSet; +use yaml_rust2::{Yaml, YamlLoader}; + use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; use super::error::DocumentationError; @@ -11,7 +13,6 @@ use super::repository_text; use manifest::tracked_scopes; const DEPENDABOT_PATH: &str = ".github/dependabot.yml"; -const UPDATE_MARKER: &str = " - package-ecosystem: "; #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] struct DependencyScope { @@ -29,17 +30,27 @@ pub(super) fn check( } fn admit(raw: &str, required: &BTreeSet) -> Result<(), DocumentationError> { - if !raw.starts_with("version: 2\nupdates:\n") { - return Err(contract("version and updates header is exact")); + let documents = + YamlLoader::load_from_str(raw).map_err(|source| DocumentationError::RepositoryYaml { + path: DEPENDABOT_PATH, + source, + })?; + let [document] = documents.as_slice() else { + return Err(contract("policy contains exactly one YAML document")); + }; + if document["version"].as_i64() != Some(2) { + return Err(contract("policy version is exactly 2")); } - let blocks = update_blocks(raw); - if blocks.is_empty() { + let Some(updates) = document["updates"].as_vec() else { + return Err(contract("updates is a sequence")); + }; + if updates.is_empty() { return Err(contract("at least one update block exists")); } let mut configured = BTreeSet::new(); - for block in blocks { - let scopes = block_scopes(&block)?; - admit_maintenance_policy(&block, &scopes)?; + for update in updates { + let scopes = block_scopes(update)?; + admit_maintenance_policy(update, &scopes)?; for scope in scopes { if !configured.insert(scope.clone()) { return Err(contract_at( @@ -58,47 +69,26 @@ fn admit(raw: &str, required: &BTreeSet) -> Result<(), Document Ok(()) } -fn update_blocks(raw: &str) -> Vec> { - let mut blocks = Vec::new(); - let mut block = Vec::new(); - let mut active = false; - for line in raw.lines() { - if line.starts_with(UPDATE_MARKER) { - if active { - blocks.push(std::mem::take(&mut block)); - } - active = true; - } - if active { - block.push(line); - } - } - if active { - blocks.push(block); - } - blocks -} - -fn block_scopes(block: &[&str]) -> Result, DocumentationError> { - let ecosystem = block - .first() - .and_then(|line| line.strip_prefix(UPDATE_MARKER)) - .map(unquote) +fn block_scopes(update: &Yaml) -> Result, DocumentationError> { + let ecosystem = update["package-ecosystem"] + .as_str() .ok_or_else(|| contract("every update block names an ecosystem"))?; let mut scopes = Vec::new(); - let mut remaining = block; - while let Some((line, rest)) = remaining.split_first() { - remaining = rest; - if let Some(directory) = line.strip_prefix(" directory: ") { - scopes.push(DependencyScope::new(ecosystem, unquote(directory))); - } else if *line == " directories:" { - while let Some((entry, rest)) = remaining.split_first() { - let Some(directory) = entry.strip_prefix(" - ") else { - break; - }; - scopes.push(DependencyScope::new(ecosystem, unquote(directory))); - remaining = rest; - } + if !update["directory"].is_badvalue() { + let directory = update["directory"] + .as_str() + .ok_or_else(|| contract("update directory is a string"))?; + scopes.push(DependencyScope::new(ecosystem, directory)); + } + if !update["directories"].is_badvalue() { + let directories = update["directories"] + .as_vec() + .ok_or_else(|| contract("update directories is a sequence"))?; + for directory in directories { + let directory = directory + .as_str() + .ok_or_else(|| contract("every update directory is a string"))?; + scopes.push(DependencyScope::new(ecosystem, directory)); } } if scopes.is_empty() { @@ -112,13 +102,15 @@ fn block_scopes(block: &[&str]) -> Result, DocumentationErr } fn admit_maintenance_policy( - block: &[&str], + update: &Yaml, scopes: &[DependencyScope], ) -> Result<(), DocumentationError> { - let raw = block.join("\n"); - let uniform = raw.contains(" schedule:\n interval: weekly") - && raw.contains(" open-pull-requests-limit: 5") - && raw.contains(" labels:\n - dependencies"); + let labels_are_exact = update["labels"].as_vec().is_some_and( + |labels| matches!(labels.as_slice(), [label] if label.as_str() == Some("dependencies")), + ); + let uniform = update["schedule"]["interval"].as_str() == Some("weekly") + && update["open-pull-requests-limit"].as_i64() == Some(5) + && labels_are_exact; if uniform { Ok(()) } else { @@ -132,18 +124,6 @@ fn admit_maintenance_policy( } } -fn unquote(raw: &str) -> &str { - let bytes = raw.as_bytes(); - match (bytes.first(), bytes.last()) { - (Some(first), Some(last)) - if bytes.len() >= 2 && first == last && matches!(first, b'\'' | b'"') => - { - raw.get(1..raw.len().saturating_sub(1)).unwrap_or(raw) - } - _ => raw, - } -} - impl DependencyScope { fn new(ecosystem: &str, directory: &str) -> Self { Self { diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs index ce56642..3daa31b 100644 --- a/xtask/src/documentation_integrity/dependabot/tests.rs +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -43,21 +43,13 @@ fn complete_uniform_dependabot_policy_is_admitted() { #[test] fn list_termination_preserves_the_following_scope_declaration() { - let block = [ - " - package-ecosystem: cargo", - " directories:", - " - /", - " directory: /xtask", - ]; + let policy = POLICY.replacen(" - /xtask\n", "", 1).replacen( + " schedule:\n", + " directory: /xtask\n schedule:\n", + 1, + ); - let scopes = super::block_scopes(&block); - assert!(matches!( - scopes, - Ok(scopes) if scopes == vec![ - DependencyScope::new("cargo", "/"), - DependencyScope::new("cargo", "/xtask"), - ] - )); + assert!(super::admit(&policy, &required()).is_ok()); } #[test] @@ -99,6 +91,28 @@ fn nonuniform_maintenance_policy_is_refused() { )); } +#[test] +fn prefix_confusable_maintenance_values_are_refused() { + for policy in [ + POLICY.replacen(" interval: weekly", " interval: weeklylies", 1), + POLICY.replacen( + " open-pull-requests-limit: 5", + " open-pull-requests-limit: 50", + 1, + ), + POLICY.replacen(" - dependencies", " - dependencies-extra", 1), + ] { + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContractAt { + path: super::DEPENDABOT_PATH, + ref subject, + requirement: "update block uses the maintenance policy", + }) if subject == "cargo /" + )); + } +} + #[test] fn committed_dependabot_policy_covers_every_tracked_manifest() -> Result<(), Box> { From c5980d87f01c044c31927ef2d02aada39b57db95 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:53:55 -0700 Subject: [PATCH 045/113] Fix: require documentation CI enforcement --- CHANGELOG.md | 3 +- .../workflow_contract.rs | 14 ++++++- .../workflow_contract/tests.rs | 42 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9788316..518f472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ after its public API and format compatibility policies are established. commands as YAML, rejects guarded or non-string `run` values, preserves declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed - documentation CI command exactly once, and applies one deadline across + documentation CI command exactly once, rejects guarded or failure-tolerant + documentation jobs and required run steps, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index fa918a0..67a98b9 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -49,7 +49,14 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let [document] = documents.as_slice() else { return Err(contract("workflow contains exactly one YAML document")); }; - let Some(steps) = document["jobs"]["documentation"]["steps"].as_vec() else { + let job = &document["jobs"]["documentation"]; + if !job["if"].is_badvalue() { + return Err(contract("documentation job is unguarded")); + } + if !job["continue-on-error"].is_badvalue() { + return Err(contract("documentation job is failure-intolerant")); + } + let Some(steps) = job["steps"].as_vec() else { return Err(contract("workflow defines documentation job steps")); }; let mut runs = Vec::new(); @@ -61,6 +68,11 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> if !step["if"].is_badvalue() { return Err(contract("documentation job run steps are unguarded")); } + if !step["continue-on-error"].is_badvalue() { + return Err(contract( + "documentation job run steps are failure-intolerant", + )); + } let Some(run) = run.as_str() else { return Err(contract("documentation job run values are strings")); }; diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 91a111c..fa682b1 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -115,6 +115,48 @@ fn guarded_required_commands_do_not_satisfy_the_contract() { )); } +#[test] +fn guarded_documentation_jobs_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace(" documentation:\n", " documentation:\n if: false\n"); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job is unguarded", + }) + )); +} + +#[test] +fn failure_tolerant_documentation_jobs_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " documentation:\n", + " documentation:\n continue-on-error: true\n", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job is failure-intolerant", + }) + )); +} + +#[test] +fn failure_tolerant_required_commands_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n continue-on-error: true\n run:", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job run steps are failure-intolerant", + }) + )); +} + #[test] fn unreviewed_python_executables_are_refused() { let workflow = WORKFLOW.replace( From cc8f72833cc4a2a3a787f9f0ea579a2a775c4563 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:56:18 -0700 Subject: [PATCH 046/113] Fix: require pinned Node CI setup --- CHANGELOG.md | 5 +- .../workflow_contract.rs | 27 +++++++ .../workflow_contract/tests.rs | 78 +++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 518f472..9d6ccfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,9 @@ after its public API and format compatibility policies are established. commands as YAML, rejects guarded or non-string `run` values, preserves declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed - documentation CI command exactly once, rejects guarded or failure-tolerant - documentation jobs and required run steps, and applies one deadline across + documentation CI command and the pinned Node setup action exactly once, + requires the reviewed Node version, rejects guarded or failure-tolerant + documentation jobs and required steps, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 67a98b9..b1d72ad 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -10,6 +10,8 @@ use super::repository_text; const CI_PATH: &str = ".github/workflows/ci.yml"; const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ documentation_integrity::execution::external_tests -- --ignored"; +const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; +const NODE_VERSION: &str = "24.18.0"; const XTASK_COMMAND: &str = "cargo xtask documentation-integrity-check"; const REVIEWED_RUNS: &[&str] = &[ "rustup show", @@ -60,7 +62,27 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> return Err(contract("workflow defines documentation job steps")); }; let mut runs = Vec::new(); + let mut node_setup_seen = false; for step in steps { + if step["uses"].as_str() == Some(SETUP_NODE_ACTION) { + if node_setup_seen { + return Err(contract( + "documentation job installs pinned Node.js exactly once", + )); + } + node_setup_seen = true; + if !step["if"].is_badvalue() { + return Err(contract("documentation Node.js setup is unguarded")); + } + if !step["continue-on-error"].is_badvalue() { + return Err(contract( + "documentation Node.js setup is failure-intolerant", + )); + } + if step["with"]["node-version"].as_str() != Some(NODE_VERSION) { + return Err(contract("documentation Node.js version is 24.18.0")); + } + } let run = &step["run"]; if run.is_badvalue() { continue; @@ -78,6 +100,11 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> }; runs.push(run.trim_end_matches('\n').to_owned()); } + if !node_setup_seen { + return Err(contract( + "documentation job installs pinned Node.js exactly once", + )); + } Ok(runs) } diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index fa682b1..f61c417 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -9,6 +9,10 @@ jobs: steps: - name: Install Rust run: rustup show + - name: Install pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.18.0 - name: Install documentation tools run: | documentation_tools="$RUNNER_TEMP/documentation-tools" @@ -69,6 +73,10 @@ fn inert_yaml_cannot_impersonate_documentation_commands() { jobs: documentation: steps: + - name: Install pinned Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24.18.0 # run: rustup show - name: "run: cargo xtask documentation-integrity-check" uses: example/action@0123456789abcdef @@ -157,6 +165,76 @@ fn failure_tolerant_required_commands_do_not_satisfy_the_contract() { )); } +#[test] +fn documentation_job_requires_the_pinned_node_action_once() { + let setup = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" + ); + let missing = WORKFLOW.replace(setup, ""); + let drifted = WORKFLOW.replace( + "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", + "actions/setup-node@0123456789abcdef0123456789abcdef01234567", + ); + + for workflow in [&missing, &drifted] { + assert!(matches!( + super::admit(workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); + } +} + +#[test] +fn guarded_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " if: false\n uses: actions/setup-node@", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation Node.js setup is unguarded", + }) + )); +} + +#[test] +fn failure_tolerant_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " continue-on-error: true\n uses: actions/setup-node@", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation Node.js setup is failure-intolerant", + }) + )); +} + +#[test] +fn documentation_job_requires_the_reviewed_node_version() { + let workflow = WORKFLOW.replace( + " node-version: 24.18.0", + " node-version: 24.18.1", + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation Node.js version is 24.18.0", + }) + )); +} + #[test] fn unreviewed_python_executables_are_refused() { let workflow = WORKFLOW.replace( From 558e464128f15dc86e8484dcf0b57ceead059db5 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 15:58:26 -0700 Subject: [PATCH 047/113] Fix: count extensionless executable sources --- CHANGELOG.md | 3 ++- xtask/src/source_structure.rs | 8 +++--- xtask/src/source_structure/pure_rust_tests.rs | 23 ++++++++++++++++ xtask/src/source_structure/python_source.rs | 26 +++++++++++++++---- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d6ccfd..709b007 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,7 +54,8 @@ after its public API and format compatibility policies are established. - Golden File Worldline verification now runs through a dependency-isolated Rust `xtask`, cross-checks every identity-bearing digest against external `b3sum`, and CI refuses Rust, Python, or shell source modules that exceed the - documented 500-physical-line hard maximum, including test modules. + documented 500-physical-line hard maximum, including test modules and + extensionless executable sources. - Repository source verification now uses capability-relative, no-follow file opens and verifies repository-root identity after Git inventory and again after source scanning, so a persistent root replacement or source path diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 4402908..9a02796 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -11,7 +11,7 @@ use std::path::Path; use crate::git_inventory::{GitPath, paths as git_paths}; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; -use python_source::refuse_executable_python; +use python_source::{FileExecution, refuse_executable_python}; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; use source_kind::{is_extensionless_file, is_python_module, is_source_candidate}; @@ -110,8 +110,10 @@ fn source_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { - refuse_executable_python(source_root, &relative)?; - if is_extensionless_file(relative.as_str().as_bytes()) { + let execution = refuse_executable_python(source_root, &relative)?; + if is_extensionless_file(relative.as_str().as_bytes()) + && execution == FileExecution::NonExecutable + { continue; } let lines = source_line_count(source_root, &relative)?; diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 2b34b57..fbe31dd 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -76,6 +76,29 @@ fn extension_bearing_executable_python_is_refused_by_the_pure_rust_boundary() Ok(()) } +#[test] +fn extensionless_executable_source_obeys_the_line_limit() -> Result<(), Box> +{ + let directory = TestDirectory::create("extensionless-source-limit")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check"); + fs::write(&script, format!("#!/bin/sh\n{}", ":\n".repeat(500)))?; + let mut permissions = fs::metadata(&script)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&script, permissions)?; + let present = BTreeSet::from([GitPath::new(b"check".to_vec())]); + + let paths = super::select_source_paths(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let violations = super::source_violations(&source_root, paths)?; + + assert_eq!(violations, vec![String::from("check")]); + drop(source_root); + directory.close()?; + Ok(()) +} + #[test] fn extensionless_nonexecutable_text_is_not_a_source_module() -> Result<(), Box> { diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 3f60191..2b43f87 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -11,10 +11,16 @@ use super::repository_path::RepositoryPath; const SHEBANG_SCAN_BYTES: u64 = 1_024; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum FileExecution { + Executable, + NonExecutable, +} + pub(super) fn refuse_executable_python( source_root: &RepositoryRoot, relative: &RepositoryPath, -) -> Result<(), SourceStructureError> { +) -> Result { let path = source_root.display_path(relative.as_path()); let file = source_root .open_file(relative.as_path()) @@ -25,23 +31,33 @@ pub(super) fn refuse_executable_python( }, OpenRepositoryFileError::NonRegular => SourceStructureError::NonRegular(path.clone()), })?; - let python = executable_uses_python(&file).map_err(|source| SourceStructureError::Inspect { + let execution = file_execution(&file).map_err(|source| SourceStructureError::Inspect { path: path.clone(), source, })?; + let python = execution == FileExecution::Executable + && executable_uses_python(file).map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; if python { Err(SourceStructureError::PythonSource( relative.as_str().to_owned(), )) } else { - Ok(()) + Ok(execution) } } -fn executable_uses_python(file: &File) -> Result { +fn file_execution(file: &File) -> Result { if file.metadata()?.permissions().mode() & 0o111 == 0 { - return Ok(false); + Ok(FileExecution::NonExecutable) + } else { + Ok(FileExecution::Executable) } +} + +fn executable_uses_python(file: File) -> Result { let mut prefix = Vec::new(); file.take(SHEBANG_SCAN_BYTES).read_to_end(&mut prefix)?; Ok(is_python_shebang(&prefix)) From cc1aaca9e23942076df27bcbc500f9d32ebd72da Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:03:41 -0700 Subject: [PATCH 048/113] Fix: parse env shebang utilities --- CHANGELOG.md | 4 +- xtask/src/source_structure/python_source.rs | 27 ++-- .../python_source/environment.rs | 149 ++++++++++++++++++ .../python_source/environment/word_split.rs | 70 ++++++++ 4 files changed, 234 insertions(+), 16 deletions(-) create mode 100644 xtask/src/source_structure/python_source/environment.rs create mode 100644 xtask/src/source_structure/python_source/environment/word_split.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 709b007..c46c6cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,9 @@ after its public API and format compatibility policies are established. after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses `.py`, `.pyw`, and Python shebangs in every executable source candidate, - including attached `env -S` interpreter strings. + including attached `env -S` interpreter strings. Environment shebangs parse + options, assignments, quoting, and split strings before classifying only the + selected utility, so later command arguments cannot impersonate Python. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 2b43f87..1cbc5ee 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -1,5 +1,7 @@ //! This module owns bounded executable Python-shebang admission. +mod environment; + use std::fs::File; use std::io::{self, Read}; use std::os::unix::fs::PermissionsExt; @@ -71,10 +73,9 @@ fn is_python_shebang(prefix: &[u8]) -> bool { else { return false; }; - let mut words = line - .split(u8::is_ascii_whitespace) - .filter(|word| !word.is_empty()); - let Some(interpreter) = words.next() else { + let line = line.trim_ascii_start(); + let mut fields = line.splitn(2, u8::is_ascii_whitespace); + let Some(interpreter) = fields.next().filter(|field| !field.is_empty()) else { return false; }; if is_python_program(interpreter) { @@ -83,17 +84,10 @@ fn is_python_shebang(prefix: &[u8]) -> bool { if !program_name(interpreter).eq_ignore_ascii_case(b"env") { return false; } - words.any(environment_word_selects_python) -} - -fn environment_word_selects_python(word: &[u8]) -> bool { - if let Some(split) = word.strip_prefix(b"--split-string=") { - return is_python_program(split); - } - if let Some(split) = word.strip_prefix(b"-S").filter(|split| !split.is_empty()) { - return is_python_program(split); - } - !word.starts_with(b"-") && !word.contains(&b'=') && is_python_program(word) + fields + .next() + .and_then(environment::selected_utility) + .is_some_and(|utility| is_python_program(&utility)) } fn is_python_program(program: &[u8]) -> bool { @@ -143,6 +137,9 @@ mod tests { for prefix in [ b"#!/bin/sh\n".as_slice(), b"#!/usr/bin/env bash\n", + b"#!/usr/bin/env sh -c python3\n", + b"#!/usr/bin/env -S sh -c 'echo python3'\n", + b"#!/usr/bin/env -S \"sh -c 'echo python3'\"\n", b"python3\n", b"first line\n#!/usr/bin/python3\n", ] { diff --git a/xtask/src/source_structure/python_source/environment.rs b/xtask/src/source_structure/python_source/environment.rs new file mode 100644 index 0000000..8de79b9 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment.rs @@ -0,0 +1,149 @@ +//! This module owns deterministic `env` shebang utility selection. + +mod word_split; + +use std::collections::VecDeque; + +use word_split::split_words; + +pub(super) fn selected_utility(arguments: &[u8]) -> Option> { + let mut words = VecDeque::from(split_words(arguments)?); + let mut options = true; + let mut split_budget = arguments.len().checked_add(1)?; + while let Some(word) = words.pop_front() { + if options { + if word == b"--" { + options = false; + continue; + } + if word == b"-" || is_flag(&word) { + continue; + } + if option_takes_value(&word) { + words.pop_front()?; + continue; + } + if option_has_value(&word) { + continue; + } + if let Some(split) = split_value(&word) { + split_budget = split_budget.checked_sub(1)?; + words = expanded_words(split, words)?; + options = true; + continue; + } + if word.starts_with(b"-") { + return None; + } + } + if word.contains(&b'=') { + options = false; + continue; + } + return Some(word); + } + None +} + +fn expanded_words(first: &[u8], remaining: VecDeque>) -> Option>> { + let mut input = first.to_vec(); + for word in remaining { + if !input.is_empty() { + input.push(b' '); + } + input.extend_from_slice(&word); + } + Some(VecDeque::from(split_words(&input)?)) +} + +fn split_value(word: &[u8]) -> Option<&[u8]> { + if word == b"-S" || word == b"--split-string" { + Some(b"") + } else { + word.strip_prefix(b"-S") + .filter(|value| !value.is_empty()) + .or_else(|| word.strip_prefix(b"--split-string=")) + } +} + +fn option_takes_value(word: &[u8]) -> bool { + matches!( + word, + b"-u" | b"--unset" | b"-C" | b"--chdir" | b"-a" | b"--argv0" + ) +} + +fn option_has_value(word: &[u8]) -> bool { + [b"-u".as_slice(), b"-C", b"-a"].iter().any(|prefix| { + word.strip_prefix(*prefix) + .is_some_and(|value| !value.is_empty()) + }) || [ + b"--unset=".as_slice(), + b"--chdir=".as_slice(), + b"--argv0=".as_slice(), + ] + .iter() + .any(|prefix| word.starts_with(prefix)) +} + +fn is_flag(word: &[u8]) -> bool { + matches!( + word, + b"--ignore-environment" + | b"--debug" + | b"--null" + | b"--help" + | b"--version" + | b"--list-signal-handling" + ) || is_short_flag_set(word) + || [ + b"--block-signal".as_slice(), + b"--default-signal", + b"--ignore-signal", + ] + .iter() + .any(|prefix| { + word == *prefix + || word + .strip_prefix(*prefix) + .is_some_and(|value| value.starts_with(b"=")) + }) +} + +fn is_short_flag_set(word: &[u8]) -> bool { + word.strip_prefix(b"-").is_some_and(|flags| { + !flags.is_empty() && flags.iter().all(|flag| matches!(flag, b'i' | b'v' | b'0')) + }) +} + +#[cfg(test)] +mod tests { + #[test] + fn options_assignments_and_split_strings_precede_the_utility() { + for arguments in [ + b"-i python3 -I".as_slice(), + b"-iv python3", + b"-u PYTHONHOME python3", + b"NAME=value python3", + b"-S -i python3 -I", + b"-S \"python3 -I\"", + b"--split-string='python3 -I'", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(b"python3".to_vec()) + ); + } + } + + #[test] + fn arguments_after_the_utility_cannot_replace_it() { + for arguments in [ + b"sh -c python3".as_slice(), + b"-S sh -c 'echo python3'", + b"-S \"sh -c 'echo python3'\"", + ] { + assert_eq!(super::selected_utility(arguments), Some(b"sh".to_vec())); + } + } +} diff --git a/xtask/src/source_structure/python_source/environment/word_split.rs b/xtask/src/source_structure/python_source/environment/word_split.rs new file mode 100644 index 0000000..b15c4e0 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/word_split.rs @@ -0,0 +1,70 @@ +//! This module owns bounded shell-word splitting for `env -S`. + +pub(super) fn split_words(input: &[u8]) -> Option>> { + let mut splitter = WordSplitter::default(); + for byte in input { + splitter.observe(*byte); + } + splitter.finish() +} + +#[derive(Default)] +struct WordSplitter { + words: Vec>, + word: Vec, + started: bool, + quote: Quote, + escaped: bool, +} + +impl WordSplitter { + fn observe(&mut self, byte: u8) { + if self.escaped { + self.word.push(byte); + self.escaped = false; + return; + } + match (self.quote, byte) { + (Quote::None, byte) if byte.is_ascii_whitespace() => self.complete_word(), + (Quote::None, b'\'') => self.start_quote(Quote::Single), + (Quote::None, b'"') => self.start_quote(Quote::Double), + (Quote::None | Quote::Double, b'\\') => { + self.escaped = true; + self.started = true; + } + (Quote::Single, b'\'') | (Quote::Double, b'"') => self.quote = Quote::None, + (_, byte) => { + self.word.push(byte); + self.started = true; + } + } + } + + fn complete_word(&mut self) { + if self.started { + self.words.push(std::mem::take(&mut self.word)); + self.started = false; + } + } + + const fn start_quote(&mut self, quote: Quote) { + self.quote = quote; + self.started = true; + } + + fn finish(mut self) -> Option>> { + if self.escaped || self.quote != Quote::None { + return None; + } + self.complete_word(); + Some(self.words) + } +} + +#[derive(Clone, Copy, Default, Eq, PartialEq)] +enum Quote { + #[default] + None, + Single, + Double, +} From 8f2a0ae957672ee769c912034df142cd15a3939b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:05:12 -0700 Subject: [PATCH 049/113] Fix: refuse dot-only Python basenames --- CHANGELOG.md | 9 +++++---- xtask/src/source_structure/pure_rust_tests.rs | 5 ++++- xtask/src/source_structure/source_kind.rs | 18 ++++++++++++++---- xtask/src/source_structure/tests.rs | 2 ++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c46c6cf..2a740d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,10 +60,11 @@ after its public API and format compatibility policies are established. opens and verifies repository-root identity after Git inventory and again after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses - `.py`, `.pyw`, and Python shebangs in every executable source candidate, - including attached `env -S` interpreter strings. Environment shebangs parse - options, assignments, quoting, and split strings before classifying only the - selected utility, so later command arguments cannot impersonate Python. + `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every + executable source candidate, including attached `env -S` interpreter + strings. Environment shebangs parse options, assignments, quoting, and split + strings before classifying only the selected utility, so later command + arguments cannot impersonate Python. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index fbe31dd..c81671d 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -11,13 +11,16 @@ use crate::test_directory::TestDirectory; #[test] fn python_source_is_refused_by_the_pure_rust_boundary() { for path in [ + ".py", + "scripts/.PYW", "scripts/check.py", "scripts/check.PY", "scripts/check.pyw", "scripts/check.PYW", ] { + let present = BTreeSet::from([GitPath::new(path.as_bytes().to_vec())]); assert!(matches!( - super::admit_source_path(&GitPath::new(path.as_bytes().to_vec())), + super::select_source_paths(&present, &BTreeSet::new()), Err(super::SourceStructureError::PythonSource(ref observed)) if observed == path )); diff --git a/xtask/src/source_structure/source_kind.rs b/xtask/src/source_structure/source_kind.rs index ee97792..2d369ff 100644 --- a/xtask/src/source_structure/source_kind.rs +++ b/xtask/src/source_structure/source_kind.rs @@ -1,14 +1,20 @@ //! This module owns repository source-module classification. pub(super) fn is_source_module(path: &[u8]) -> bool { + if is_python_module(path) { + return true; + } let Some(suffix) = source_suffix(path) else { return false; }; - suffix == b"rs" || suffix == b"sh" || is_python_suffix(suffix) + suffix == b"rs" || suffix == b"sh" } pub(super) fn is_python_module(path: &[u8]) -> bool { - source_suffix(path).is_some_and(is_python_suffix) + repository_file_name(path).is_some_and(|file_name| { + file_name.strip_prefix(b".").is_some_and(is_python_suffix) + || source_suffix(path).is_some_and(is_python_suffix) + }) } pub(super) fn is_source_candidate(path: &[u8]) -> bool { @@ -16,7 +22,7 @@ pub(super) fn is_source_candidate(path: &[u8]) -> bool { } pub(super) fn is_extensionless_file(path: &[u8]) -> bool { - let Some(file_name) = path.rsplit(|byte| *byte == b'/').next() else { + let Some(file_name) = repository_file_name(path) else { return false; }; !file_name.is_empty() && !file_name.contains(&b'.') @@ -27,9 +33,13 @@ const fn is_python_suffix(suffix: &[u8]) -> bool { } fn source_suffix(path: &[u8]) -> Option<&[u8]> { - let file_name = path.rsplit(|byte| *byte == b'/').next()?; + let file_name = repository_file_name(path)?; let mut components = file_name.rsplitn(2, |byte| *byte == b'.'); let suffix = components.next()?; let stem = components.next()?; (!stem.is_empty()).then_some(suffix) } + +fn repository_file_name(path: &[u8]) -> Option<&[u8]> { + path.rsplit(|byte| *byte == b'/').next() +} diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index 44f79f0..8966ab4 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -274,6 +274,8 @@ fn source_module_limit_accepts_five_hundred_and_refuses_five_hundred_one() { #[test] fn source_module_classification_is_explicit() { assert!(is_source_module(b"src/lib.rs")); + assert!(is_source_module(b".py")); + assert!(is_source_module(b"scripts/.PYW")); assert!(is_source_module(b"scripts/check.py")); assert!(is_source_module(b"scripts/check.pyw")); assert!(is_source_module(b"scripts/check.sh")); From e5189268ea5a372e8223c4b12619aa0f438067a0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:07:29 -0700 Subject: [PATCH 050/113] Fix: refuse alternate Node setup actions --- CHANGELOG.md | 7 +-- .../workflow_contract.rs | 7 ++- .../workflow_contract/tests.rs | 49 +++++++++++++++---- 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a740d6..ea8d713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,10 @@ after its public API and format compatibility policies are established. declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed documentation CI command and the pinned Node setup action exactly once, - requires the reviewed Node version, rejects guarded or failure-tolerant - documentation jobs and required steps, and applies one deadline across - captured and inherited child execution and output collection. + refuses alternate setup-node actions, requires the reviewed Node version, + rejects guarded or failure-tolerant documentation jobs and required steps, + and applies one deadline across captured and inherited child execution and + output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation Git inventory and tools start from one retained repository directory handle, diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index b1d72ad..b0059ef 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -11,6 +11,7 @@ const CI_PATH: &str = ".github/workflows/ci.yml"; const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ documentation_integrity::execution::external_tests -- --ignored"; const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; +const SETUP_NODE_ACTION_PREFIX: &str = "actions/setup-node@"; const NODE_VERSION: &str = "24.18.0"; const XTASK_COMMAND: &str = "cargo xtask documentation-integrity-check"; const REVIEWED_RUNS: &[&str] = &[ @@ -64,7 +65,11 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let mut runs = Vec::new(); let mut node_setup_seen = false; for step in steps { - if step["uses"].as_str() == Some(SETUP_NODE_ACTION) { + let action = step["uses"].as_str(); + if action.is_some_and(|value| value.starts_with(SETUP_NODE_ACTION_PREFIX)) { + if action != Some(SETUP_NODE_ACTION) { + return Err(contract("documentation Node.js setup action is pinned")); + } if node_setup_seen { return Err(contract( "documentation job installs pinned Node.js exactly once", diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index f61c417..0df5b0b 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -174,20 +174,49 @@ fn documentation_job_requires_the_pinned_node_action_once() { " node-version: 24.18.0\n" ); let missing = WORKFLOW.replace(setup, ""); - let drifted = WORKFLOW.replace( + assert!(matches!( + super::admit(&missing), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); +} + +#[test] +fn drifted_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", "actions/setup-node@0123456789abcdef0123456789abcdef01234567", ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); +} - for workflow in [&missing, &drifted] { - assert!(matches!( - super::admit(workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation job installs pinned Node.js exactly once", - }) - )); - } +#[test] +fn additional_unpinned_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Install documentation tools\n", + concat!( + " - name: Replace Node.js\n", + " uses: actions/setup-node@0123456789abcdef0123456789abcdef01234567\n", + " with:\n", + " node-version: 24.18.1\n", + " - name: Install documentation tools\n" + ), + ); + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); } #[test] From 53fcd72f10eed08557c66a6b49ee8225ea614617 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:09:35 -0700 Subject: [PATCH 051/113] Fix: split documentation workflow admission --- .../workflow_contract.rs | 85 ++++++++++++------- xtask/tests/documentation_cli_contract.rs | 18 ++++ 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index b0059ef..2e74243 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -1,6 +1,6 @@ //! This module owns the CI documentation-job execution contract. -use yaml_rust2::YamlLoader; +use yaml_rust2::{Yaml, YamlLoader}; use crate::repository_file::RepositoryRoot; @@ -52,6 +52,10 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let [document] = documents.as_slice() else { return Err(contract("workflow contains exactly one YAML document")); }; + reviewed_runs(documentation_steps(document)?) +} + +fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError> { let job = &document["jobs"]["documentation"]; if !job["if"].is_badvalue() { return Err(contract("documentation job is unguarded")); @@ -62,48 +66,22 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let Some(steps) = job["steps"].as_vec() else { return Err(contract("workflow defines documentation job steps")); }; + Ok(steps) +} + +fn reviewed_runs(steps: &[Yaml]) -> Result, DocumentationError> { let mut runs = Vec::new(); let mut node_setup_seen = false; for step in steps { - let action = step["uses"].as_str(); - if action.is_some_and(|value| value.starts_with(SETUP_NODE_ACTION_PREFIX)) { - if action != Some(SETUP_NODE_ACTION) { - return Err(contract("documentation Node.js setup action is pinned")); - } + if admit_node_setup(step)?.is_some() { if node_setup_seen { return Err(contract( "documentation job installs pinned Node.js exactly once", )); } node_setup_seen = true; - if !step["if"].is_badvalue() { - return Err(contract("documentation Node.js setup is unguarded")); - } - if !step["continue-on-error"].is_badvalue() { - return Err(contract( - "documentation Node.js setup is failure-intolerant", - )); - } - if step["with"]["node-version"].as_str() != Some(NODE_VERSION) { - return Err(contract("documentation Node.js version is 24.18.0")); - } - } - let run = &step["run"]; - if run.is_badvalue() { - continue; - } - if !step["if"].is_badvalue() { - return Err(contract("documentation job run steps are unguarded")); } - if !step["continue-on-error"].is_badvalue() { - return Err(contract( - "documentation job run steps are failure-intolerant", - )); - } - let Some(run) = run.as_str() else { - return Err(contract("documentation job run values are strings")); - }; - runs.push(run.trim_end_matches('\n').to_owned()); + runs.extend(admit_run(step)?); } if !node_setup_seen { return Err(contract( @@ -113,6 +91,47 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> Ok(runs) } +fn admit_node_setup(step: &Yaml) -> Result, DocumentationError> { + let action = step["uses"].as_str(); + if !action.is_some_and(|value| value.starts_with(SETUP_NODE_ACTION_PREFIX)) { + return Ok(None); + } + if action != Some(SETUP_NODE_ACTION) { + return Err(contract("documentation Node.js setup action is pinned")); + } + if !step["if"].is_badvalue() { + return Err(contract("documentation Node.js setup is unguarded")); + } + if !step["continue-on-error"].is_badvalue() { + return Err(contract( + "documentation Node.js setup is failure-intolerant", + )); + } + if step["with"]["node-version"].as_str() != Some(NODE_VERSION) { + return Err(contract("documentation Node.js version is 24.18.0")); + } + Ok(Some(())) +} + +fn admit_run(step: &Yaml) -> Result, DocumentationError> { + let run = &step["run"]; + if run.is_badvalue() { + return Ok(None); + } + if !step["if"].is_badvalue() { + return Err(contract("documentation job run steps are unguarded")); + } + if !step["continue-on-error"].is_badvalue() { + return Err(contract( + "documentation job run steps are failure-intolerant", + )); + } + let Some(run) = run.as_str() else { + return Err(contract("documentation job run values are strings")); + }; + Ok(Some(run.trim_end_matches('\n').to_owned())) +} + fn runs_are_reviewed(runs: &[String]) -> bool { runs.len() == REVIEWED_RUNS.len() && REVIEWED_RUNS diff --git a/xtask/tests/documentation_cli_contract.rs b/xtask/tests/documentation_cli_contract.rs index 8d088a7..74251c1 100644 --- a/xtask/tests/documentation_cli_contract.rs +++ b/xtask/tests/documentation_cli_contract.rs @@ -9,6 +9,7 @@ use std::io; const DOCUMENTATION_ERROR_DISPLAY: &str = include_str!("../src/documentation_integrity/error/display.rs"); +const WORKFLOW_CONTRACT: &str = include_str!("../src/documentation_integrity/workflow_contract.rs"); #[test] fn documentation_error_formatter_stays_below_the_hard_function_limit() -> Result<(), &'static str> { @@ -25,6 +26,23 @@ fn documentation_error_formatter_stays_below_the_hard_function_limit() -> Result Ok(()) } +#[test] +fn workflow_parser_stays_below_the_hard_function_limit() -> Result<(), &'static str> { + let (_, after_signature) = WORKFLOW_CONTRACT + .split_once( + "fn documentation_runs(workflow: &str) -> Result, DocumentationError> {", + ) + .ok_or("workflow contract must retain its parser")?; + let (body, _) = after_signature + .split_once("\n}\n\nfn documentation_steps") + .ok_or("workflow parser must remain a directly inspectable function")?; + assert!( + body.lines().count() <= 59, + "documentation_runs exceeds the 60-line hard limit" + ); + Ok(()) +} + #[test] fn successful_verification_runs_every_documentation_tool_silently() -> Result<(), io::Error> { let tools = documentation_tools::DocumentationTools::create()?; From 4e942a6efd1e0359e672075634a8555588851229 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:11:23 -0700 Subject: [PATCH 052/113] Fix: isolate Node workflow regressions --- .../workflow_contract/node_setup.rs | 103 ++++++++++++++++++ .../workflow_contract/tests.rs | 101 +---------------- 2 files changed, 105 insertions(+), 99 deletions(-) create mode 100644 xtask/src/documentation_integrity/workflow_contract/node_setup.rs diff --git a/xtask/src/documentation_integrity/workflow_contract/node_setup.rs b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs new file mode 100644 index 0000000..61dec4e --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs @@ -0,0 +1,103 @@ +//! This module owns documentation-job Node.js setup policy regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +#[test] +fn documentation_job_requires_the_pinned_node_action_once() { + let setup = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" + ); + let missing = WORKFLOW.replace(setup, ""); + assert!(matches!( + admit(&missing), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); +} + +#[test] +fn drifted_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", + "actions/setup-node@0123456789abcdef0123456789abcdef01234567", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); +} + +#[test] +fn additional_unpinned_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Install documentation tools\n", + concat!( + " - name: Replace Node.js\n", + " uses: actions/setup-node@0123456789abcdef0123456789abcdef01234567\n", + " with:\n", + " node-version: 24.18.1\n", + " - name: Install documentation tools\n" + ), + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup action is pinned", + }) + )); +} + +#[test] +fn guarded_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " if: false\n uses: actions/setup-node@", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup is unguarded", + }) + )); +} + +#[test] +fn failure_tolerant_node_setup_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " uses: actions/setup-node@", + " continue-on-error: true\n uses: actions/setup-node@", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js setup is failure-intolerant", + }) + )); +} + +#[test] +fn documentation_job_requires_the_reviewed_node_version() { + let workflow = WORKFLOW.replace( + " node-version: 24.18.0", + " node-version: 24.18.1", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation Node.js version is 24.18.0", + }) + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 0df5b0b..31f43d8 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -2,6 +2,8 @@ use std::path::Path; use crate::repository_file::RepositoryRoot; +mod node_setup; + const WORKFLOW: &str = r#"name: CI jobs: documentation: @@ -165,105 +167,6 @@ fn failure_tolerant_required_commands_do_not_satisfy_the_contract() { )); } -#[test] -fn documentation_job_requires_the_pinned_node_action_once() { - let setup = concat!( - " - name: Install pinned Node.js\n", - " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", - " with:\n", - " node-version: 24.18.0\n" - ); - let missing = WORKFLOW.replace(setup, ""); - assert!(matches!( - super::admit(&missing), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation job installs pinned Node.js exactly once", - }) - )); -} - -#[test] -fn drifted_node_setup_does_not_satisfy_the_contract() { - let workflow = WORKFLOW.replace( - "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020", - "actions/setup-node@0123456789abcdef0123456789abcdef01234567", - ); - assert!(matches!( - super::admit(&workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation Node.js setup action is pinned", - }) - )); -} - -#[test] -fn additional_unpinned_node_setup_does_not_satisfy_the_contract() { - let workflow = WORKFLOW.replace( - " - name: Install documentation tools\n", - concat!( - " - name: Replace Node.js\n", - " uses: actions/setup-node@0123456789abcdef0123456789abcdef01234567\n", - " with:\n", - " node-version: 24.18.1\n", - " - name: Install documentation tools\n" - ), - ); - assert!(matches!( - super::admit(&workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation Node.js setup action is pinned", - }) - )); -} - -#[test] -fn guarded_node_setup_does_not_satisfy_the_contract() { - let workflow = WORKFLOW.replace( - " uses: actions/setup-node@", - " if: false\n uses: actions/setup-node@", - ); - assert!(matches!( - super::admit(&workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation Node.js setup is unguarded", - }) - )); -} - -#[test] -fn failure_tolerant_node_setup_does_not_satisfy_the_contract() { - let workflow = WORKFLOW.replace( - " uses: actions/setup-node@", - " continue-on-error: true\n uses: actions/setup-node@", - ); - assert!(matches!( - super::admit(&workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation Node.js setup is failure-intolerant", - }) - )); -} - -#[test] -fn documentation_job_requires_the_reviewed_node_version() { - let workflow = WORKFLOW.replace( - " node-version: 24.18.0", - " node-version: 24.18.1", - ); - assert!(matches!( - super::admit(&workflow), - Err(super::DocumentationError::RepositoryContract { - path: super::CI_PATH, - requirement: "documentation Node.js version is 24.18.0", - }) - )); -} - #[test] fn unreviewed_python_executables_are_refused() { let workflow = WORKFLOW.replace( From 40968fa6abef2e851bf08dda97a5a2fe9f664e3b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:34:53 -0700 Subject: [PATCH 053/113] Fix: admit documentation workflow actions --- CHANGELOG.md | 6 +- .../workflow_contract.rs | 106 ++++++++++++++---- .../workflow_contract/action_steps.rs | 81 +++++++++++++ .../workflow_contract/tests.rs | 10 +- 4 files changed, 179 insertions(+), 24 deletions(-) create mode 100644 xtask/src/documentation_integrity/workflow_contract/action_steps.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ea8d713..dc12436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,10 @@ after its public API and format compatibility policies are established. declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed documentation CI command and the pinned Node setup action exactly once, - refuses alternate setup-node actions, requires the reviewed Node version, - rejects guarded or failure-tolerant documentation jobs and required steps, + admits only the exact pinned checkout and Node setup actions in their reviewed + order, rejects checkout overrides and unreviewed action steps, refuses + alternate setup-node actions, requires the reviewed Node version, rejects + guarded or failure-tolerant documentation jobs and required steps, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 2e74243..64a43b3 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -8,6 +8,8 @@ use super::error::DocumentationError; use super::repository_text; const CI_PATH: &str = ".github/workflows/ci.yml"; +const CHECKOUT_ACTION: &str = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; +const CHECKOUT_ACTION_PREFIX: &str = "actions/checkout@"; const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ documentation_integrity::execution::external_tests -- --ignored"; const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; @@ -71,46 +73,99 @@ fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError fn reviewed_runs(steps: &[Yaml]) -> Result, DocumentationError> { let mut runs = Vec::new(); - let mut node_setup_seen = false; + let mut actions = Vec::new(); for step in steps { - if admit_node_setup(step)?.is_some() { - if node_setup_seen { - return Err(contract( - "documentation job installs pinned Node.js exactly once", - )); - } - node_setup_seen = true; - } + actions.extend(admit_action(step)?); runs.extend(admit_run(step)?); } - if !node_setup_seen { + if actions + .iter() + .filter(|action| **action == DocumentationAction::Node) + .count() + != 1 + { return Err(contract( "documentation job installs pinned Node.js exactly once", )); } + if actions.as_slice() != REVIEWED_ACTIONS { + return Err(contract( + "documentation job actions execute in reviewed order", + )); + } Ok(runs) } -fn admit_node_setup(step: &Yaml) -> Result, DocumentationError> { - let action = step["uses"].as_str(); - if !action.is_some_and(|value| value.starts_with(SETUP_NODE_ACTION_PREFIX)) { +fn admit_action(step: &Yaml) -> Result, DocumentationError> { + let uses = &step["uses"]; + if uses.is_badvalue() { return Ok(None); } - if action != Some(SETUP_NODE_ACTION) { + let Some(action) = uses.as_str() else { + return Err(contract("documentation job action values are strings")); + }; + if action.starts_with(CHECKOUT_ACTION_PREFIX) { + return admit_checkout(step, action).map(Some); + } + if action.starts_with(SETUP_NODE_ACTION_PREFIX) { + return admit_node_setup(step, action).map(Some); + } + Err(contract("documentation job action steps are reviewed")) +} + +fn admit_checkout(step: &Yaml, action: &str) -> Result { + if action != CHECKOUT_ACTION { + return Err(contract("documentation checkout action is pinned")); + } + admit_action_execution( + step, + "documentation checkout is unguarded", + "documentation checkout is failure-intolerant", + )?; + let exact = step["with"] + .as_hash() + .is_some_and(|configuration| configuration.len() == 1) + && step["with"]["persist-credentials"].as_bool() == Some(false); + if !exact { + return Err(contract("documentation checkout configuration is exact")); + } + Ok(DocumentationAction::Checkout) +} + +fn admit_node_setup(step: &Yaml, action: &str) -> Result { + if action != SETUP_NODE_ACTION { return Err(contract("documentation Node.js setup action is pinned")); } + admit_action_execution( + step, + "documentation Node.js setup is unguarded", + "documentation Node.js setup is failure-intolerant", + )?; + let exact = step["with"] + .as_hash() + .is_some_and(|configuration| configuration.len() == 1) + && step["with"]["node-version"].as_str() == Some(NODE_VERSION); + if !exact { + return Err(contract("documentation Node.js version is 24.18.0")); + } + Ok(DocumentationAction::Node) +} + +fn admit_action_execution( + step: &Yaml, + guard_requirement: &'static str, + failure_requirement: &'static str, +) -> Result<(), DocumentationError> { if !step["if"].is_badvalue() { - return Err(contract("documentation Node.js setup is unguarded")); + return Err(contract(guard_requirement)); } if !step["continue-on-error"].is_badvalue() { - return Err(contract( - "documentation Node.js setup is failure-intolerant", - )); + return Err(contract(failure_requirement)); } - if step["with"]["node-version"].as_str() != Some(NODE_VERSION) { - return Err(contract("documentation Node.js version is 24.18.0")); + if !step["run"].is_badvalue() { + return Err(contract("documentation action steps do not define run")); } - Ok(Some(())) + Ok(()) } fn admit_run(step: &Yaml) -> Result, DocumentationError> { @@ -139,6 +194,15 @@ fn runs_are_reviewed(runs: &[String]) -> bool { .all(|required| runs.iter().filter(|run| run.as_str() == *required).count() == 1) } +const REVIEWED_ACTIONS: &[DocumentationAction] = + &[DocumentationAction::Checkout, DocumentationAction::Node]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DocumentationAction { + Checkout, + Node, +} + const fn contract(requirement: &'static str) -> DocumentationError { DocumentationError::RepositoryContract { path: CI_PATH, diff --git a/xtask/src/documentation_integrity/workflow_contract/action_steps.rs b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs new file mode 100644 index 0000000..077cc2c --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs @@ -0,0 +1,81 @@ +//! This module owns documentation-job action-step policy regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +const CHECKOUT_STEP: &str = concat!( + " - name: Check out repository\n", + " uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1\n", + " with:\n", + " persist-credentials: false\n" +); +const NODE_STEP: &str = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" +); + +#[test] +fn checkout_of_an_unreviewed_revision_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " persist-credentials: false", + " persist-credentials: false\n ref: main", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation checkout configuration is exact", + }) + )); +} + +#[test] +fn drifted_checkout_action_does_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "actions/checkout@0123456789abcdef0123456789abcdef01234567", + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation checkout action is pinned", + }) + )); +} + +#[test] +fn unreviewed_action_steps_do_not_satisfy_the_contract() { + let workflow = WORKFLOW.replace( + " - name: Install Rust\n", + concat!( + " - name: Unreviewed action\n", + " uses: example/action@0123456789abcdef0123456789abcdef01234567\n", + " - name: Install Rust\n" + ), + ); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job action steps are reviewed", + }) + )); +} + +#[test] +fn documentation_actions_execute_in_reviewed_order() { + let workflow = WORKFLOW + .replace(CHECKOUT_STEP, "CHECKOUT_STEP_PLACEHOLDER\n") + .replace(NODE_STEP, CHECKOUT_STEP) + .replace("CHECKOUT_STEP_PLACEHOLDER\n", NODE_STEP); + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job actions execute in reviewed order", + }) + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 31f43d8..3fd2ad0 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -2,6 +2,7 @@ use std::path::Path; use crate::repository_file::RepositoryRoot; +mod action_steps; mod node_setup; const WORKFLOW: &str = r#"name: CI @@ -9,6 +10,10 @@ jobs: documentation: name: Documentation steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Install Rust run: rustup show - name: Install pinned Node.js @@ -75,13 +80,16 @@ fn inert_yaml_cannot_impersonate_documentation_commands() { jobs: documentation: steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false - name: Install pinned Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: 24.18.0 # run: rustup show - name: "run: cargo xtask documentation-integrity-check" - uses: example/action@0123456789abcdef "#; assert!(matches!( super::admit(workflow), From d331f36be229bddb63985b970034f3231cb54769 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:39:01 -0700 Subject: [PATCH 054/113] Fix: synchronize descendant readiness --- xtask/src/bounded_process/process_group.rs | 57 +++++++++++---- .../process_group/child_tests.rs | 69 +++++++++++-------- .../process_group/readiness_tests.rs | 42 +++++++++++ .../bounded_process/process_group/tests.rs | 25 ++++--- 4 files changed, 145 insertions(+), 48 deletions(-) create mode 100644 xtask/src/bounded_process/process_group/readiness_tests.rs diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs index 558033b..0f8ad08 100644 --- a/xtask/src/bounded_process/process_group.rs +++ b/xtask/src/bounded_process/process_group.rs @@ -3,6 +3,13 @@ use std::io; use std::process::Child; +#[cfg(test)] +use std::io::Read; +#[cfg(test)] +use std::os::unix::net::UnixListener; +#[cfg(test)] +use std::path::Path; + use rustix::io::Errno; use rustix::process::{Pid, Signal, kill_process_group}; @@ -28,6 +35,8 @@ impl ProcessGroup { #[cfg(test)] const DESCENDANT_CHILD: &str = "KEEP_XTASK_DESCENDANT_CHILD"; #[cfg(test)] +const DESCENDANT_CHILD_READY: &str = "KEEP_XTASK_DESCENDANT_CHILD_READY"; +#[cfg(test)] const DESCENDANT_PARENT: &str = "KEEP_XTASK_DESCENDANT_PARENT"; #[cfg(test)] const DESCENDANT_READY: &str = "KEEP_XTASK_DESCENDANT_READY"; @@ -37,26 +46,50 @@ const DESCENDANT_SOCKET: &str = "KEEP_XTASK_DESCENDANT_SOCKET"; const INTERRUPT_SUPERVISOR: &str = "KEEP_XTASK_INTERRUPT_SUPERVISOR"; #[cfg(test)] -fn wait_for_ready(path: &std::path::Path) -> Result<(), io::Error> { - let expires = std::time::Instant::now() - .checked_add(std::time::Duration::from_secs(2)) - .ok_or_else(|| io::Error::other("descendant readiness deadline overflow"))?; - while !path.is_file() { - if std::time::Instant::now() >= expires { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "descendant did not become ready", - )); +fn readiness_listener(path: &Path) -> Result { + let listener = UnixListener::bind(path)?; + listener.set_nonblocking(true)?; + Ok(listener) +} + +#[cfg(test)] +fn wait_for_ready(listener: &UnixListener, child: &mut Child) -> Result<(), io::Error> { + loop { + match listener.accept() { + Ok((mut stream, _address)) => { + stream.set_nonblocking(false)?; + let mut signal = [0_u8; 1]; + stream.read_exact(&mut signal)?; + if signal != [b'r'] { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "descendant sent an invalid readiness signal", + )); + } + return Ok(()); + } + Err(source) if source.kind() == io::ErrorKind::WouldBlock => { + if let Some(status) = child.try_wait()? { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("child exited before descendant readiness: {status}"), + )); + } + std::thread::yield_now(); + } + Err(source) => return Err(source), } - std::thread::yield_now(); } - Ok(()) } #[cfg(test)] #[path = "process_group/child_tests.rs"] mod child_tests; +#[cfg(test)] +#[path = "process_group/readiness_tests.rs"] +mod readiness_tests; + #[cfg(test)] #[path = "process_group/tests.rs"] mod tests; diff --git a/xtask/src/bounded_process/process_group/child_tests.rs b/xtask/src/bounded_process/process_group/child_tests.rs index 51327d2..f2adb43 100644 --- a/xtask/src/bounded_process/process_group/child_tests.rs +++ b/xtask/src/bounded_process/process_group/child_tests.rs @@ -1,15 +1,13 @@ //! This module owns subprocess fixtures for process-group regression tests. use std::env; -use std::fs; -use std::io; -use std::os::unix::net::UnixListener; -use std::path::Path; +use std::io::{self, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; use std::process::Command; use super::{ - DESCENDANT_CHILD, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, - wait_for_ready, + DESCENDANT_CHILD, DESCENDANT_CHILD_READY, DESCENDANT_PARENT, DESCENDANT_READY, + DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, readiness_listener, wait_for_ready, }; use crate::bounded_process::{ProcessError, capture}; @@ -26,6 +24,15 @@ fn process_supervisor_captures_descendant_until_interrupted() -> Result<(), io:: "bounded_process::process_group::child_tests::process_child_leaves_descendant_pipe_open", ]) .env(DESCENDANT_PARENT, "1") + .env( + DESCENDANT_CHILD_READY, + env::var_os(DESCENDANT_CHILD_READY).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant child-ready path", + ) + })?, + ) .env( DESCENDANT_READY, env::var_os(DESCENDANT_READY).ok_or_else(|| { @@ -61,26 +68,33 @@ fn process_child_leaves_descendant_pipe_open() -> Result<(), io::Error> { let ready = env::var_os(DESCENDANT_READY).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") })?; - drop( - Command::new(executable) - .args([ - "--exact", - "bounded_process::process_group::child_tests::process_descendant_holds_pipe", - ]) - .env(DESCENDANT_CHILD, "1") - .env(DESCENDANT_READY, &ready) - .env( - DESCENDANT_SOCKET, - env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "missing descendant socket path", - ) - })?, - ) - .spawn()?, - ); - wait_for_ready(Path::new(&ready))?; + let child_ready = env::var_os(DESCENDANT_CHILD_READY).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant child-ready path", + ) + })?; + let listener = readiness_listener(std::path::Path::new(&child_ready))?; + let mut descendant = Command::new(executable) + .args([ + "--exact", + "bounded_process::process_group::child_tests::process_descendant_holds_pipe", + ]) + .env(DESCENDANT_CHILD, "1") + .env(DESCENDANT_READY, &child_ready) + .env( + DESCENDANT_SOCKET, + env::var_os(DESCENDANT_SOCKET).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "missing descendant socket path", + ) + })?, + ) + .spawn()?; + wait_for_ready(&listener, &mut descendant)?; + let mut readiness = UnixStream::connect(ready)?; + readiness.write_all(b"r")?; Ok(()) } @@ -99,7 +113,8 @@ fn process_descendant_holds_pipe() -> Result<(), io::Error> { io::Error::new(io::ErrorKind::InvalidInput, "missing descendant ready path") })?; let listener = UnixListener::bind(socket)?; - fs::write(ready, b"ready")?; + let mut readiness = UnixStream::connect(ready)?; + readiness.write_all(b"r")?; loop { let (mut stream, _) = listener.accept()?; let mut command = [0_u8; 1]; diff --git a/xtask/src/bounded_process/process_group/readiness_tests.rs b/xtask/src/bounded_process/process_group/readiness_tests.rs new file mode 100644 index 0000000..207bbf5 --- /dev/null +++ b/xtask/src/bounded_process/process_group/readiness_tests.rs @@ -0,0 +1,42 @@ +//! This module owns deterministic readiness synchronization evidence. + +use std::env; +use std::io; +use std::process::{Command, Stdio}; + +use super::{readiness_listener, wait_for_ready}; +use crate::test_directory::TestDirectory; + +#[test] +fn readiness_wait_does_not_depend_on_wall_clock() { + let process_group = include_str!("../process_group.rs"); + + assert!( + !process_group.contains("std::time::"), + "readiness must be bounded by the child lifecycle, not wall-clock time" + ); +} + +#[test] +fn readiness_refuses_child_exit_without_a_signal() -> Result<(), Box> { + let directory = TestDirectory::create("readiness-exit")?; + let listener = readiness_listener(&directory.path().join("ready"))?; + let mut child = Command::new(env::current_exe()?) + .args([ + "--exact", + "bounded_process::process_group::readiness_tests::readiness_wait_does_not_depend_on_wall_clock", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + let result = wait_for_ready(&listener, &mut child); + directory.close()?; + + assert!(matches!( + result, + Err(source) if source.kind() == io::ErrorKind::UnexpectedEof + )); + Ok(()) +} diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 672b2e9..65bd24c 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -12,7 +12,8 @@ use std::time::{Duration, Instant}; use rustix::process::{Pid, Signal, kill_process}; use super::{ - DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, INTERRUPT_SUPERVISOR, wait_for_ready, + DESCENDANT_CHILD_READY, DESCENDANT_PARENT, DESCENDANT_READY, DESCENDANT_SOCKET, + INTERRUPT_SUPERVISOR, readiness_listener, wait_for_ready, }; use crate::bounded_process::cleanup::cleanup_process; use crate::bounded_process::{ProcessError, capture_with}; @@ -25,11 +26,14 @@ const SUPERVISOR_PROCESS: &str = "bounded_process::process_group::child_tests::p fn spawn_ready_descendant( executable: &Path, ready: &Path, + child_ready: &Path, socket: &Path, ) -> Result> { + let listener = readiness_listener(ready)?; let mut child = Command::new(executable) .args(["--exact", CHILD_PROCESS]) .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_CHILD_READY, child_ready) .env(DESCENDANT_READY, ready) .env(DESCENDANT_SOCKET, socket) .stdin(Stdio::null()) @@ -37,7 +41,7 @@ fn spawn_ready_descendant( .stderr(Stdio::piped()) .process_group(0) .spawn()?; - if let Err(source) = wait_for_ready(ready) { + if let Err(source) = wait_for_ready(&listener, &mut child) { let error = cleanup_process( &mut child, ProcessError::Io { @@ -56,18 +60,21 @@ fn terminal_interrupt_terminates_the_isolated_descendant_group() -> Result<(), Box> { let directory = TestDirectory::create("process-group-interrupt")?; let ready = directory.path().join("ready"); + let child_ready = directory.path().join("child-ready"); let socket = directory.path().join("descendant.sock"); + let listener = readiness_listener(&ready)?; let executable = env::current_exe()?; let mut supervisor = Command::new(executable) .args(["--exact", SUPERVISOR_PROCESS]) .env(INTERRUPT_SUPERVISOR, "1") + .env(DESCENDANT_CHILD_READY, &child_ready) .env(DESCENDANT_READY, &ready) .env(DESCENDANT_SOCKET, &socket) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn()?; - wait_for_ready(&ready)?; + wait_for_ready(&listener, &mut supervisor)?; let supervisor_pid = Pid::from_raw(i32::try_from(supervisor.id())?).ok_or("supervisor process ID is zero")?; kill_process(supervisor_pid, Signal::INT)?; @@ -91,14 +98,11 @@ fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box Result<(), Box Result<(), Box> { let directory = TestDirectory::create("process-group-cleanup")?; let ready = directory.path().join("ready"); + let child_ready = directory.path().join("child-ready"); let socket = directory.path().join("descendant.sock"); + let listener = readiness_listener(&ready)?; let executable = env::current_exe()?; let mut command = Command::new(executable); command .args(["--exact", CHILD_PROCESS]) .env(DESCENDANT_PARENT, "1") + .env(DESCENDANT_CHILD_READY, &child_ready) .env(DESCENDANT_READY, &ready) .env(DESCENDANT_SOCKET, &socket) .stdin(Stdio::null()) @@ -135,7 +142,7 @@ fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box Date: Tue, 28 Jul 2026 16:48:28 -0700 Subject: [PATCH 055/113] Fix: inspect every executable for Python --- CHANGELOG.md | 9 +- docs/Rust Standards.md | 4 +- xtask/src/source_structure.rs | 76 +++++--------- .../executable_candidate_tests.rs | 80 +++++++++++++++ xtask/src/source_structure/python_source.rs | 27 +++-- xtask/src/source_structure/source_error.rs | 4 +- .../src/source_structure/source_inventory.rs | 99 +++++++++++++++++++ 7 files changed, 224 insertions(+), 75 deletions(-) create mode 100644 xtask/src/source_structure/executable_candidate_tests.rs create mode 100644 xtask/src/source_structure/source_inventory.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc12436..3123210 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,10 +64,11 @@ after its public API and format compatibility policies are established. after source scanning, so a persistent root replacement or source path replaced with a symlink is refused. The pure Rust boundary also refuses `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every - executable source candidate, including attached `env -S` interpreter - strings. Environment shebangs parse options, assignments, quoting, and split - strings before classifying only the selected utility, so later command - arguments cannot impersonate Python. + executable regular file regardless of filename suffix, including raw + non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment + shebangs parse options, assignments, quoting, and split strings before + classifying only the selected utility, so later command arguments cannot + impersonate Python. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 2d666be..6af4a1c 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -2057,8 +2057,8 @@ Also forbidden: - hidden filesystem access; - hidden network access; - hidden allocation proportional to input; -- Python source files, including `.py`, `.pyw`, and extensionless executable - Python shebangs; +- Python source files, including `.py`, `.pyw`, and executable Python shebangs + in regular files regardless of filename suffix; - random IDs where content identity is required; - wall-clock time in deterministic algorithms; - hashing arbitrary serializer output; diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 9a02796..3cc2047 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -3,27 +3,24 @@ mod python_source; mod repository_path; mod source_error; +mod source_inventory; mod source_kind; -use std::collections::BTreeSet; use std::io::{self, BufRead, BufReader}; use std::path::Path; -use crate::git_inventory::{GitPath, paths as git_paths}; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; use python_source::{FileExecution, refuse_executable_python}; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; -use source_kind::{is_extensionless_file, is_python_module, is_source_candidate}; +#[cfg(test)] +use source_inventory::{ + PRESENT_PATH_ARGUMENTS, select as select_source_inventory, select_source_paths, +}; +use source_inventory::{SourceInventory, collect as source_paths}; +use source_kind::is_extensionless_file; const SOURCE_MODULE_HARD_LIMIT_LINES: u64 = 500; -const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", -]; pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> { let source_root = @@ -33,7 +30,7 @@ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> })?; let paths = source_paths(repository_root)?; verify_source_root(&source_root, repository_root)?; - let violations = source_violations(&source_root, paths)?; + let violations = inventory_violations(&source_root, paths)?; verify_source_root(&source_root, repository_root)?; if violations.is_empty() { Ok(()) @@ -61,47 +58,14 @@ fn verify_source_root( } } -fn source_paths(repository_root: &Path) -> Result, SourceStructureError> { - let present = git_paths( - repository_root, - &PRESENT_PATH_ARGUMENTS, - "git ls-files present", - )?; - let deleted = git_paths( - repository_root, - &["ls-files", "-z", "--deleted"], - "git ls-files deleted", - )?; - select_source_paths(&present, &deleted) -} - -fn select_source_paths( - present: &BTreeSet, - deleted: &BTreeSet, -) -> Result, SourceStructureError> { - present - .difference(deleted) - .filter(|path| is_source_candidate(path.as_bytes())) - .map(admit_source_path) - .collect() -} - -fn admit_source_path(path: &GitPath) -> Result { - let python = is_python_module(path.as_bytes()); - let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { - SourceStructureError::GitPathEncoding { - operation: "source path admission", - source, - } - })?; - let relative = RepositoryPath::admit(text)?; - if python { - Err(SourceStructureError::PythonSource( - relative.as_str().to_owned(), - )) - } else { - Ok(relative) +fn inventory_violations( + source_root: &RepositoryRoot, + inventory: SourceInventory, +) -> Result, SourceStructureError> { + for relative in inventory.executable_candidates { + let _execution = refuse_executable_python(source_root, &relative)?; } + source_violations(source_root, inventory.modules) } fn source_violations( @@ -110,7 +74,12 @@ fn source_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { - let execution = refuse_executable_python(source_root, &relative)?; + let execution = refuse_executable_python(source_root, relative.as_path())?; + if execution == FileExecution::NonRegular { + return Err(SourceStructureError::NonRegular( + source_root.display_path(relative.as_path()), + )); + } if is_extensionless_file(relative.as_str().as_bytes()) && execution == FileExecution::NonExecutable { @@ -213,6 +182,9 @@ impl LineCounter { } } +#[cfg(test)] +#[path = "source_structure/executable_candidate_tests.rs"] +mod executable_candidate_tests; #[cfg(test)] #[path = "source_structure/pure_rust_tests.rs"] mod pure_rust_tests; diff --git a/xtask/src/source_structure/executable_candidate_tests.rs b/xtask/src/source_structure/executable_candidate_tests.rs new file mode 100644 index 0000000..99a6c9c --- /dev/null +++ b/xtask/src/source_structure/executable_candidate_tests.rs @@ -0,0 +1,80 @@ +//! This module owns executable admission outside known source suffixes. + +use std::collections::BTreeSet; +use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use crate::git_inventory::GitPath; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +#[test] +fn non_source_suffix_cannot_hide_executable_python() -> Result<(), Box> { + let directory = TestDirectory::create("hidden-python")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check.txt"); + fs::write(&script, b"#!/usr/bin/python3\nprint('forbidden')\n")?; + make_executable(&script)?; + let present = BTreeSet::from([GitPath::new(b"check.txt".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let result = super::inventory_violations(&source_root, paths); + + assert!(matches!( + result, + Err(super::SourceStructureError::PythonSource(ref path)) + if path == Path::new("check.txt") + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn non_executable_text_remains_outside_the_python_boundary() +-> Result<(), Box> { + let directory = TestDirectory::create("non-executable-text")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + fs::write( + repository.join("check.txt"), + b"#!/usr/bin/python3\nnot executable\n", + )?; + let present = BTreeSet::from([GitPath::new(b"check.txt".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let violations = super::inventory_violations(&source_root, paths)?; + + assert!(violations.is_empty()); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn non_utf8_executable_candidate_preserves_path_bytes() -> Result<(), Box> { + let path_bytes = b"check-\xff.txt"; + let present = BTreeSet::from([GitPath::new(path_bytes.to_vec())]); + + let inventory = super::select_source_inventory(&present, &BTreeSet::new())?; + + assert_eq!( + inventory + .executable_candidates + .first() + .map(|path| path.as_os_str().as_bytes()), + Some(path_bytes.as_slice()) + ); + Ok(()) +} + +fn make_executable(path: &Path) -> Result<(), std::io::Error> { + let mut permissions = fs::metadata(path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions) +} diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 1cbc5ee..067b8fa 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -9,7 +9,7 @@ use std::os::unix::fs::PermissionsExt; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; use super::SourceStructureError; -use super::repository_path::RepositoryPath; +use std::path::Path; const SHEBANG_SCAN_BYTES: u64 = 1_024; @@ -17,22 +17,21 @@ const SHEBANG_SCAN_BYTES: u64 = 1_024; pub(super) enum FileExecution { Executable, NonExecutable, + NonRegular, } pub(super) fn refuse_executable_python( source_root: &RepositoryRoot, - relative: &RepositoryPath, + relative: &Path, ) -> Result { - let path = source_root.display_path(relative.as_path()); - let file = source_root - .open_file(relative.as_path()) - .map_err(|error| match error { - OpenRepositoryFileError::Io(source) => SourceStructureError::Inspect { - path: path.clone(), - source, - }, - OpenRepositoryFileError::NonRegular => SourceStructureError::NonRegular(path.clone()), - })?; + let path = source_root.display_path(relative); + let file = match source_root.open_file(relative) { + Ok(file) => file, + Err(OpenRepositoryFileError::NonRegular) => return Ok(FileExecution::NonRegular), + Err(OpenRepositoryFileError::Io(source)) => { + return Err(SourceStructureError::Inspect { path, source }); + } + }; let execution = file_execution(&file).map_err(|source| SourceStructureError::Inspect { path: path.clone(), source, @@ -43,9 +42,7 @@ pub(super) fn refuse_executable_python( source, })?; if python { - Err(SourceStructureError::PythonSource( - relative.as_str().to_owned(), - )) + Err(SourceStructureError::PythonSource(relative.to_owned())) } else { Ok(execution) } diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 5f583ba..30162de 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -21,7 +21,7 @@ pub(crate) enum SourceStructureError { }, InvalidPath(String), NonRegular(PathBuf), - PythonSource(String), + PythonSource(PathBuf), RepositoryRootChanged(PathBuf), Violations { maximum: u64, @@ -59,7 +59,7 @@ impl fmt::Display for SourceStructureError { } Self::PythonSource(path) => { formatter.write_str("pure Rust source boundary refuses Python module `")?; - escaped_controls(formatter, path)?; + escaped_path(formatter, path)?; formatter.write_str("`") } Self::RepositoryRootChanged(path) => { diff --git a/xtask/src/source_structure/source_inventory.rs b/xtask/src/source_structure/source_inventory.rs new file mode 100644 index 0000000..2d06483 --- /dev/null +++ b/xtask/src/source_structure/source_inventory.rs @@ -0,0 +1,99 @@ +//! This module owns deterministic source and executable-candidate inventory. + +use std::collections::BTreeSet; +use std::ffi::OsString; +use std::os::unix::ffi::OsStringExt; +use std::path::{Component, Path, PathBuf}; + +use crate::git_inventory::{GitPath, paths as git_paths}; + +use super::repository_path::RepositoryPath; +use super::source_error::SourceStructureError; +use super::source_kind::{is_python_module, is_source_candidate}; + +pub(super) const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", +]; + +pub(super) struct SourceInventory { + pub(super) modules: Vec, + pub(super) executable_candidates: Vec, +} + +pub(super) fn collect(repository_root: &Path) -> Result { + let present = git_paths( + repository_root, + &PRESENT_PATH_ARGUMENTS, + "git ls-files present", + )?; + let deleted = git_paths( + repository_root, + &["ls-files", "-z", "--deleted"], + "git ls-files deleted", + )?; + select(&present, &deleted) +} + +pub(super) fn select( + present: &BTreeSet, + deleted: &BTreeSet, +) -> Result { + let modules = select_source_paths(present, deleted)?; + let executable_candidates = present + .difference(deleted) + .filter(|path| !is_source_candidate(path.as_bytes())) + .map(admit_inspection_path) + .collect::, _>>()?; + Ok(SourceInventory { + modules, + executable_candidates, + }) +} + +pub(super) fn select_source_paths( + present: &BTreeSet, + deleted: &BTreeSet, +) -> Result, SourceStructureError> { + present + .difference(deleted) + .filter(|path| is_source_candidate(path.as_bytes())) + .map(admit_source_path) + .collect() +} + +fn admit_source_path(path: &GitPath) -> Result { + let python = is_python_module(path.as_bytes()); + let text = path_text(path, "source path admission")?; + let relative = RepositoryPath::admit(text)?; + if python { + Err(SourceStructureError::PythonSource( + relative.as_path().to_owned(), + )) + } else { + Ok(relative) + } +} + +fn admit_inspection_path(path: &GitPath) -> Result { + let relative = PathBuf::from(OsString::from_vec(path.as_bytes().to_vec())); + if relative + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + Ok(relative) + } else { + Err(SourceStructureError::InvalidPath(path_text( + path, + "executable path admission", + )?)) + } +} + +fn path_text(path: &GitPath, operation: &'static str) -> Result { + String::from_utf8(path.as_bytes().to_vec()) + .map_err(|source| SourceStructureError::GitPathEncoding { operation, source }) +} From b64ec3915f9790ff9292e9bb39d8167800ff98a3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 16:53:09 -0700 Subject: [PATCH 056/113] Fix: pin documentation execution context --- CHANGELOG.md | 4 +- docs/Documentation Standards.md | 6 +- .../workflow_contract.rs | 36 ++++++++++ .../workflow_contract/execution_context.rs | 71 +++++++++++++++++++ .../workflow_contract/tests.rs | 8 ++- 5 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 xtask/src/documentation_integrity/workflow_contract/execution_context.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3123210..2f9aae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,9 @@ after its public API and format compatibility policies are established. admits only the exact pinned checkout and Node setup actions in their reviewed order, rejects checkout overrides and unreviewed action steps, refuses alternate setup-node actions, requires the reviewed Node version, rejects - guarded or failure-tolerant documentation jobs and required steps, + unreviewed workflow/job run defaults and step execution fields, pins the + documentation runner and job deadline, rejects guarded or failure-tolerant + documentation jobs and required steps, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index f36ea26..21d618b 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -553,7 +553,11 @@ manifest coverage, and the documentation job's delegation to this command. The dedicated `documentation` job in `.github/workflows/ci.yml` installs the pinned tools, runs malformed-input refusal laws and the repository-owned command, and verifies repository whitespace before admitting the result as CI -evidence. +evidence. The Rust workflow contract pins that job to `ubuntu-latest` with a +ten-minute deadline, rejects workflow and job run defaults, and admits only the +reviewed action and command step fields. A custom shell, working directory, +environment, or other unreviewed execution modifier cannot impersonate a +required command. CI SHOULD block on facts it can determine reliably: diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 64a43b3..b6223cd 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -10,6 +10,10 @@ use super::repository_text; const CI_PATH: &str = ".github/workflows/ci.yml"; const CHECKOUT_ACTION: &str = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; const CHECKOUT_ACTION_PREFIX: &str = "actions/checkout@"; +const DOCUMENTATION_JOB_FIELDS: &[&str] = &["name", "runs-on", "timeout-minutes", "steps"]; +const DOCUMENTATION_JOB_NAME: &str = "Documentation and workflow integrity"; +const DOCUMENTATION_RUNNER: &str = "ubuntu-latest"; +const DOCUMENTATION_TIMEOUT_MINUTES: i64 = 10; const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ documentation_integrity::execution::external_tests -- --ignored"; const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; @@ -58,6 +62,11 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> } fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError> { + if !document["defaults"].is_badvalue() || !document["env"].is_badvalue() { + return Err(contract( + "workflow does not override documentation execution", + )); + } let job = &document["jobs"]["documentation"]; if !job["if"].is_badvalue() { return Err(contract("documentation job is unguarded")); @@ -65,6 +74,18 @@ fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError if !job["continue-on-error"].is_badvalue() { return Err(contract("documentation job is failure-intolerant")); } + if job["name"].as_str() != Some(DOCUMENTATION_JOB_NAME) { + return Err(contract("documentation job name is reviewed")); + } + if job["runs-on"].as_str() != Some(DOCUMENTATION_RUNNER) { + return Err(contract("documentation job uses ubuntu-latest")); + } + if job["timeout-minutes"].as_i64() != Some(DOCUMENTATION_TIMEOUT_MINUTES) { + return Err(contract("documentation job timeout is ten minutes")); + } + if !mapping_has_exact_fields(job, DOCUMENTATION_JOB_FIELDS) { + return Err(contract("documentation job fields are reviewed")); + } let Some(steps) = job["steps"].as_vec() else { return Err(contract("workflow defines documentation job steps")); }; @@ -165,6 +186,9 @@ fn admit_action_execution( if !step["run"].is_badvalue() { return Err(contract("documentation action steps do not define run")); } + if !mapping_has_exact_fields(step, &["name", "uses", "with"]) { + return Err(contract("documentation action step fields are reviewed")); + } Ok(()) } @@ -181,6 +205,9 @@ fn admit_run(step: &Yaml) -> Result, DocumentationError> { "documentation job run steps are failure-intolerant", )); } + if !mapping_has_exact_fields(step, &["name", "run"]) { + return Err(contract("documentation job run step fields are reviewed")); + } let Some(run) = run.as_str() else { return Err(contract("documentation job run values are strings")); }; @@ -194,6 +221,15 @@ fn runs_are_reviewed(runs: &[String]) -> bool { .all(|required| runs.iter().filter(|run| run.as_str() == *required).count() == 1) } +fn mapping_has_exact_fields(mapping: &Yaml, fields: &[&str]) -> bool { + mapping.as_hash().is_some_and(|mapping| { + mapping.len() == fields.len() + && mapping + .keys() + .all(|field| field.as_str().is_some_and(|field| fields.contains(&field))) + }) +} + const REVIEWED_ACTIONS: &[DocumentationAction] = &[DocumentationAction::Checkout, DocumentationAction::Node]; diff --git a/xtask/src/documentation_integrity/workflow_contract/execution_context.rs b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs new file mode 100644 index 0000000..902a50e --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs @@ -0,0 +1,71 @@ +//! This module owns documentation-job execution-context regressions. + +use super::super::{CI_PATH, DocumentationError, admit}; +use super::WORKFLOW; + +#[test] +fn custom_run_shell_cannot_impersonate_a_required_command() { + let workflow = WORKFLOW.replace( + " - name: Verify\n run:", + " - name: Verify\n shell: \"echo {0}\"\n run:", + ); + + assert_contract(&workflow, "documentation job run step fields are reviewed"); +} + +#[test] +fn workflow_run_defaults_cannot_replace_required_commands() { + let workflow = WORKFLOW.replace( + "jobs:\n", + "defaults:\n run:\n shell: \"echo {0}\"\njobs:\n", + ); + + assert_contract( + &workflow, + "workflow does not override documentation execution", + ); +} + +#[test] +fn job_run_defaults_cannot_replace_required_commands() { + let workflow = WORKFLOW.replace( + " steps:\n", + " defaults:\n run:\n shell: \"echo {0}\"\n steps:\n", + ); + + assert_contract(&workflow, "documentation job fields are reviewed"); +} + +#[test] +fn documentation_job_cannot_move_to_an_unreviewed_runner() { + let workflow = WORKFLOW.replace("runs-on: ubuntu-latest", "runs-on: self-hosted"); + + assert_contract(&workflow, "documentation job uses ubuntu-latest"); +} + +#[test] +fn documentation_job_cannot_extend_its_reviewed_deadline() { + let workflow = WORKFLOW.replace("timeout-minutes: 10", "timeout-minutes: 60"); + + assert_contract(&workflow, "documentation job timeout is ten minutes"); +} + +#[test] +fn action_environment_cannot_change_reviewed_execution() { + let workflow = WORKFLOW.replace( + " - name: Install pinned Node.js\n uses:", + " - name: Install pinned Node.js\n env:\n NODE_OPTIONS: --require=./hook.js\n uses:", + ); + + assert_contract(&workflow, "documentation action step fields are reviewed"); +} + +fn assert_contract(workflow: &str, requirement: &'static str) { + assert!(matches!( + admit(workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: observed, + }) if observed == requirement + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 3fd2ad0..a74b515 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -3,12 +3,15 @@ use std::path::Path; use crate::repository_file::RepositoryRoot; mod action_steps; +mod execution_context; mod node_setup; const WORKFLOW: &str = r#"name: CI jobs: documentation: - name: Documentation + name: Documentation and workflow integrity + runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 @@ -79,6 +82,9 @@ fn inert_yaml_cannot_impersonate_documentation_commands() { let workflow = r#"name: CI jobs: documentation: + name: Documentation and workflow integrity + runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 From 41faf44e6e162798b9a8f58d30d05fa54de2091c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:26:50 -0700 Subject: [PATCH 057/113] Fix: assert the contributor contract refusal --- xtask/src/documentation_integrity/contributor_contract.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xtask/src/documentation_integrity/contributor_contract.rs b/xtask/src/documentation_integrity/contributor_contract.rs index e41871e..cf454f3 100644 --- a/xtask/src/documentation_integrity/contributor_contract.rs +++ b/xtask/src/documentation_integrity/contributor_contract.rs @@ -67,7 +67,13 @@ mod tests { "git diff --cached --check\n", ); - assert!(super::admit("guide.md", invalid).is_err()); + assert!(matches!( + super::admit("guide.md", invalid), + Err(super::DocumentationError::RepositoryContract { + path: "guide.md", + requirement: "contributor command does not replace change checks with a whole-tree check", + }) + )); } #[test] From 587e4aa91df2fb132eb165026fdede0506f70b9d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:27:58 -0700 Subject: [PATCH 058/113] Fix: prove duplicate Node setup refusal --- .../workflow_contract/node_setup.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/xtask/src/documentation_integrity/workflow_contract/node_setup.rs b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs index 61dec4e..f119476 100644 --- a/xtask/src/documentation_integrity/workflow_contract/node_setup.rs +++ b/xtask/src/documentation_integrity/workflow_contract/node_setup.rs @@ -21,6 +21,25 @@ fn documentation_job_requires_the_pinned_node_action_once() { )); } +#[test] +fn duplicate_pinned_node_setup_is_refused_for_multiplicity() { + let setup = concat!( + " - name: Install pinned Node.js\n", + " uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020\n", + " with:\n", + " node-version: 24.18.0\n" + ); + let workflow = WORKFLOW.replace(setup, &setup.repeat(2)); + + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job installs pinned Node.js exactly once", + }) + )); +} + #[test] fn drifted_node_setup_does_not_satisfy_the_contract() { let workflow = WORKFLOW.replace( From e8c19479a79d67b31cb04f713d84cd1e1f419a01 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:28:40 -0700 Subject: [PATCH 059/113] Fix: cover interruption rustdoc --- xtask/tests/source_policy_contract.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 2dd75d5..c0fd0cc 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -73,6 +73,7 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " Additional {", " Cleanup {", " Io {", + " Interrupted {", " MissingStream {", " OutputLimit {", " ReaderPanic {", From 83a54cf2ed7a26ee41fe01da148806fc13973b93 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:29:35 -0700 Subject: [PATCH 060/113] Fix: test mixed-case Python extensions --- xtask/src/source_structure/pure_rust_tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index c81671d..58b5179 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -14,8 +14,10 @@ fn python_source_is_refused_by_the_pure_rust_boundary() { ".py", "scripts/.PYW", "scripts/check.py", + "scripts/check.Py", "scripts/check.PY", "scripts/check.pyw", + "scripts/check.pYw", "scripts/check.PYW", ] { let present = BTreeSet::from([GitPath::new(path.as_bytes().to_vec())]); From c4c604a6827008cce00ae71af45f05928bf45c5f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:30:42 -0700 Subject: [PATCH 061/113] Fix: reject ambiguous Dependabot directories --- CHANGELOG.md | 3 ++- xtask/src/documentation_integrity/dependabot.rs | 9 +++++++-- xtask/src/documentation_integrity/dependabot/tests.rs | 10 ++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f9aae1..5c227cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ after its public API and format compatibility policies are established. alternate setup-node actions, requires the reviewed Node version, rejects unreviewed workflow/job run defaults and step execution fields, pins the documentation runner and job deadline, rejects guarded or failure-tolerant - documentation jobs and required steps, + documentation jobs and required steps, requires each Dependabot update block + to choose exactly one directory field form, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index de566fd..f731bb1 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -73,14 +73,19 @@ fn block_scopes(update: &Yaml) -> Result, DocumentationErro let ecosystem = update["package-ecosystem"] .as_str() .ok_or_else(|| contract("every update block names an ecosystem"))?; + let has_directory = !update["directory"].is_badvalue(); + let has_directories = !update["directories"].is_badvalue(); + if has_directory && has_directories { + return Err(contract("update block chooses one directory form")); + } let mut scopes = Vec::new(); - if !update["directory"].is_badvalue() { + if has_directory { let directory = update["directory"] .as_str() .ok_or_else(|| contract("update directory is a string"))?; scopes.push(DependencyScope::new(ecosystem, directory)); } - if !update["directories"].is_badvalue() { + if has_directories { let directories = update["directories"] .as_vec() .ok_or_else(|| contract("update directories is a sequence"))?; diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs index 3daa31b..c5cf8ce 100644 --- a/xtask/src/documentation_integrity/dependabot/tests.rs +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -42,14 +42,20 @@ fn complete_uniform_dependabot_policy_is_admitted() { } #[test] -fn list_termination_preserves_the_following_scope_declaration() { +fn update_block_with_both_directory_forms_is_refused() { let policy = POLICY.replacen(" - /xtask\n", "", 1).replacen( " schedule:\n", " directory: /xtask\n schedule:\n", 1, ); - assert!(super::admit(&policy, &required()).is_ok()); + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryContract { + path: super::DEPENDABOT_PATH, + requirement: "update block chooses one directory form", + }) + )); } #[test] From bda678b37bbb31f99c40d1368d08d2e3cf6f5e11 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:31:41 -0700 Subject: [PATCH 062/113] Fix: resolve process test utilities through PATH --- repository-process-spawn/tests/working_directory.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository-process-spawn/tests/working_directory.rs b/repository-process-spawn/tests/working_directory.rs index 46d291b..da471c7 100644 --- a/repository-process-spawn/tests/working_directory.rs +++ b/repository-process-spawn/tests/working_directory.rs @@ -24,7 +24,7 @@ fn child_uses_the_opened_directory_without_mutating_parent_state() fs::create_dir(&root)?; fs::write(root.join("marker"), b"substitute\n")?; - let mut command = Command::new("/bin/cat"); + let mut command = Command::new("cat"); command.arg("marker"); set_working_directory(&mut command, directory); let output = command.output()?; @@ -46,7 +46,7 @@ fn non_directory_descriptor_refuses_before_exec() -> Result<(), Box Date: Tue, 28 Jul 2026 17:32:56 -0700 Subject: [PATCH 063/113] Fix: document bounded process boundaries --- xtask/src/bounded_process/interrupt.rs | 12 ++++++++++ xtask/src/bounded_process/process_group.rs | 9 +++++++ xtask/src/bounded_process/reader.rs | 10 ++++++++ xtask/tests/source_policy_contract.rs | 28 ++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/xtask/src/bounded_process/interrupt.rs b/xtask/src/bounded_process/interrupt.rs index fd4f978..a243fd1 100644 --- a/xtask/src/bounded_process/interrupt.rs +++ b/xtask/src/bounded_process/interrupt.rs @@ -15,6 +15,10 @@ const HANDLED_SIGNALS: [i32; 4] = [SIGINT, SIGTERM, SIGHUP, SIGQUIT]; static CONTROLLER: OnceLock = OnceLock::new(); static CONTROLLER_START: Mutex<()> = Mutex::new(()); +/// Registers one active child operation for typed terminal-signal refusal. +/// +/// Dropping an unobserved guard restores the operating system's default +/// handling for the pending signal rather than silently consuming it. pub(super) struct InterruptGuard { registry: Arc>>>, state: Arc, @@ -40,10 +44,18 @@ enum DispatchOutcome { } impl InterruptGuard { + /// Registers `program` with the shared signal controller. + /// + /// Initialization and registry-lock failures remain typed process I/O + /// failures. Registration does not block on signal delivery. pub(super) fn begin(program: &'static str) -> Result { interrupt_controller(program)?.register(program) } + /// Returns the first terminal signal observed for this operation. + /// + /// Observation is nonblocking. The returned refusal consumes the guard's + /// obligation to restore default handling for that signal on drop. pub(super) fn refusal(&self, program: &'static str) -> Option { let signal = self.state.observe()?; let signal_name = signal_hook::low_level::signal_name(signal).unwrap_or("unknown signal"); diff --git a/xtask/src/bounded_process/process_group.rs b/xtask/src/bounded_process/process_group.rs index 0f8ad08..78f264d 100644 --- a/xtask/src/bounded_process/process_group.rs +++ b/xtask/src/bounded_process/process_group.rs @@ -13,9 +13,14 @@ use std::path::Path; use rustix::io::Errno; use rustix::process::{Pid, Signal, kill_process_group}; +/// The dedicated process-group identity established for one spawned child. pub(super) struct ProcessGroup(Pid); impl ProcessGroup { + /// Admits the child's nonzero operating-system identifier as a group ID. + /// + /// Conversion fails when the unsigned child ID does not fit the platform's + /// signed PID representation or when the observed ID is zero. pub(super) fn for_child(child: &Child) -> Result { let raw = i32::try_from(child.id()) .map_err(|source| io::Error::new(io::ErrorKind::InvalidData, source))?; @@ -24,6 +29,10 @@ impl ProcessGroup { Ok(Self(pid)) } + /// Sends `SIGKILL` to every member of the admitted process group. + /// + /// An absent group is already terminated and succeeds. Other operating + /// system errors are preserved. pub(super) fn terminate(self) -> Result<(), io::Error> { match kill_process_group(self.0, Signal::KILL) { Ok(()) | Err(Errno::SRCH) => Ok(()), diff --git a/xtask/src/bounded_process/reader.rs b/xtask/src/bounded_process/reader.rs index f6d937d..c1f423d 100644 --- a/xtask/src/bounded_process/reader.rs +++ b/xtask/src/bounded_process/reader.rs @@ -10,6 +10,7 @@ use super::{InterruptGuard, ProcessDeadline, ProcessError}; const INTERRUPT_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); +/// Owns one bounded stream reader and its single-result channel. pub(super) struct ReaderWorker { handle: JoinHandle<()>, program: &'static str, @@ -18,6 +19,10 @@ pub(super) struct ReaderWorker { } impl ReaderWorker { + /// Starts a named reader that retains at most `maximum` stream bytes. + /// + /// Thread creation failure remains a typed process I/O error. The worker + /// sends exactly one bounded result and performs no unbounded buffering. pub(super) fn start( program: &'static str, stream: &'static str, @@ -43,6 +48,10 @@ impl ReaderWorker { }) } + /// Waits for the bounded result while polling the shared deadline and signal guard. + /// + /// This call blocks only for the smaller of the remaining deadline and the + /// fixed interrupt interval, so timeout and interruption remain observable. pub(super) fn receive( &self, deadline: &ProcessDeadline, @@ -72,6 +81,7 @@ impl ReaderWorker { } } + /// Joins the completed reader and maps a worker panic to a typed failure. pub(super) fn join(self) -> Result<(), ProcessError> { self.handle .join() diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index c0fd0cc..e84abe7 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -4,6 +4,9 @@ const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); const BOUNDED_PROCESS: &str = include_str!("../src/bounded_process.rs"); const BOUNDED_PROCESS_CAPTURE: &str = include_str!("../src/bounded_process/capture.rs"); const BOUNDED_PROCESS_ERROR: &str = include_str!("../src/bounded_process/error.rs"); +const BOUNDED_PROCESS_INTERRUPT: &str = include_str!("../src/bounded_process/interrupt.rs"); +const BOUNDED_PROCESS_GROUP: &str = include_str!("../src/bounded_process/process_group.rs"); +const BOUNDED_PROCESS_READER: &str = include_str!("../src/bounded_process/reader.rs"); const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs"); @@ -81,6 +84,31 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " pub(crate) fn is_not_found(", ], )?; + require_docs( + BOUNDED_PROCESS_INTERRUPT, + &[ + "pub(super) struct InterruptGuard", + " pub(super) fn begin(", + " pub(super) fn refusal(", + ], + )?; + require_docs( + BOUNDED_PROCESS_GROUP, + &[ + "pub(super) struct ProcessGroup", + " pub(super) fn for_child(", + " pub(super) fn terminate(", + ], + )?; + require_docs( + BOUNDED_PROCESS_READER, + &[ + "pub(super) struct ReaderWorker", + " pub(super) fn start(", + " pub(super) fn receive(", + " pub(super) fn join(", + ], + )?; require_docs( GIT_INVENTORY_ERROR, &[ From e046b02f9f4cd54d529dee82c8d8ae596639049e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:34:54 -0700 Subject: [PATCH 064/113] Fix: centralize executable source fixtures --- xtask/src/source_structure/pure_rust_tests.rs | 134 +++++++++++------- 1 file changed, 83 insertions(+), 51 deletions(-) diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 58b5179..80d1b87 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use std::fs; use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; use crate::git_inventory::GitPath; use crate::repository_file::RepositoryRoot; @@ -32,18 +33,15 @@ fn python_source_is_refused_by_the_pure_rust_boundary() { #[test] fn extensionless_executable_python_is_refused_by_the_pure_rust_boundary() -> Result<(), Box> { - let directory = TestDirectory::create("extensionless-python")?; - let repository = directory.path().join("repository"); - fs::create_dir(&repository)?; - let script = repository.join("check"); - fs::write(&script, b"#!/usr/bin/env python3\nprint('forbidden')\n")?; - let mut permissions = fs::metadata(&script)?.permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions)?; - let present = BTreeSet::from([GitPath::new(b"check".to_vec())]); - - let paths = super::select_source_paths(&present, &BTreeSet::new())?; - let source_root = RepositoryRoot::open(&repository)?; + let fixture = SourceFixture::create( + "extensionless-python", + "check", + b"#!/usr/bin/env python3\nprint('forbidden')\n", + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; let result = super::source_violations(&source_root, paths); assert!(matches!( @@ -51,25 +49,22 @@ fn extensionless_executable_python_is_refused_by_the_pure_rust_boundary() Err(super::SourceStructureError::PythonSource(ref path)) if path == "check" )); drop(source_root); - directory.close()?; + fixture.close()?; Ok(()) } #[test] fn extension_bearing_executable_python_is_refused_by_the_pure_rust_boundary() -> Result<(), Box> { - let directory = TestDirectory::create("extension-bearing-python")?; - let repository = directory.path().join("repository"); - fs::create_dir(&repository)?; - let script = repository.join("check.sh"); - fs::write(&script, b"#!/usr/bin/python3\nprint('forbidden')\n")?; - let mut permissions = fs::metadata(&script)?.permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions)?; - let present = BTreeSet::from([GitPath::new(b"check.sh".to_vec())]); - - let paths = super::select_source_paths(&present, &BTreeSet::new())?; - let source_root = RepositoryRoot::open(&repository)?; + let fixture = SourceFixture::create( + "extension-bearing-python", + "check.sh", + b"#!/usr/bin/python3\nprint('forbidden')\n", + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; let result = super::source_violations(&source_root, paths); assert!(matches!( @@ -77,53 +72,90 @@ fn extension_bearing_executable_python_is_refused_by_the_pure_rust_boundary() Err(super::SourceStructureError::PythonSource(ref path)) if path == "check.sh" )); drop(source_root); - directory.close()?; + fixture.close()?; Ok(()) } #[test] fn extensionless_executable_source_obeys_the_line_limit() -> Result<(), Box> { - let directory = TestDirectory::create("extensionless-source-limit")?; - let repository = directory.path().join("repository"); - fs::create_dir(&repository)?; - let script = repository.join("check"); - fs::write(&script, format!("#!/bin/sh\n{}", ":\n".repeat(500)))?; - let mut permissions = fs::metadata(&script)?.permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&script, permissions)?; - let present = BTreeSet::from([GitPath::new(b"check".to_vec())]); - - let paths = super::select_source_paths(&present, &BTreeSet::new())?; - let source_root = RepositoryRoot::open(&repository)?; + let limit = usize::try_from(super::SOURCE_MODULE_HARD_LIMIT_LINES)?; + let contents = format!("#!/bin/sh\n{}", ":\n".repeat(limit)); + let fixture = SourceFixture::create( + "extensionless-source-limit", + "check", + contents.as_bytes(), + FixtureMode::Executable, + )?; + + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; let violations = super::source_violations(&source_root, paths)?; assert_eq!(violations, vec![String::from("check")]); drop(source_root); - directory.close()?; + fixture.close()?; Ok(()) } #[test] fn extensionless_nonexecutable_text_is_not_a_source_module() -> Result<(), Box> { - let directory = TestDirectory::create("extensionless-text")?; - let repository = directory.path().join("repository"); - fs::create_dir(&repository)?; - fs::write( - repository.join("NOTICE"), + let fixture = SourceFixture::create( + "extensionless-text", + "NOTICE", b"#!/usr/bin/env python3\nnot executable\n", + FixtureMode::NonExecutable, )?; - let mut permissions = fs::metadata(repository.join("NOTICE"))?.permissions(); - permissions.set_mode(0o644); - fs::set_permissions(repository.join("NOTICE"), permissions)?; - let present = BTreeSet::from([GitPath::new(b"NOTICE".to_vec())]); - let paths = super::select_source_paths(&present, &BTreeSet::new())?; - let source_root = RepositoryRoot::open(&repository)?; + let paths = super::select_source_paths(&fixture.present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&fixture.repository)?; assert!(super::source_violations(&source_root, paths)?.is_empty()); drop(source_root); - directory.close()?; + fixture.close()?; Ok(()) } + +#[derive(Clone, Copy)] +enum FixtureMode { + Executable, + NonExecutable, +} + +struct SourceFixture { + directory: TestDirectory, + present: BTreeSet, + repository: PathBuf, +} + +impl SourceFixture { + fn create( + case: &str, + path: &str, + contents: &[u8], + mode: FixtureMode, + ) -> Result> { + let directory = TestDirectory::create(case)?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join(path); + fs::write(&script, contents)?; + let mut permissions = fs::metadata(&script)?.permissions(); + permissions.set_mode(match mode { + FixtureMode::Executable => 0o755, + FixtureMode::NonExecutable => 0o644, + }); + fs::set_permissions(&script, permissions)?; + Ok(Self { + directory, + present: BTreeSet::from([GitPath::new(path.as_bytes().to_vec())]), + repository, + }) + } + + fn close(self) -> Result<(), Box> { + self.directory.close()?; + Ok(()) + } +} From b416a2b42ba5d185f530210dbc0559d67bcbd435 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:36:19 -0700 Subject: [PATCH 065/113] Fix: fail closed on unresolved shebang utilities --- CHANGELOG.md | 2 +- xtask/src/source_structure/python_source.rs | 16 ++++++--- .../python_source/environment.rs | 35 ++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c227cd..6345a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,7 +71,7 @@ after its public API and format compatibility policies are established. non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment shebangs parse options, assignments, quoting, and split strings before classifying only the selected utility, so later command arguments cannot - impersonate Python. + impersonate Python and unresolved utility substitutions fail closed. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 067b8fa..6ef8edf 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -81,10 +81,11 @@ fn is_python_shebang(prefix: &[u8]) -> bool { if !program_name(interpreter).eq_ignore_ascii_case(b"env") { return false; } - fields - .next() - .and_then(environment::selected_utility) - .is_some_and(|utility| is_python_program(&utility)) + match fields.next().and_then(environment::selected_utility) { + Some(environment::UtilitySelection::Known(utility)) => is_python_program(&utility), + Some(environment::UtilitySelection::Ambiguous) => true, + None => false, + } } fn is_python_program(program: &[u8]) -> bool { @@ -143,4 +144,11 @@ mod tests { assert!(!is_python_shebang(prefix)); } } + + #[test] + fn unresolved_environment_utility_substitution_fails_closed() { + assert!(is_python_shebang( + b"#!/usr/bin/env -S '${UNSET_INTERPRETER}sh'\n" + )); + } } diff --git a/xtask/src/source_structure/python_source/environment.rs b/xtask/src/source_structure/python_source/environment.rs index 8de79b9..f3087c0 100644 --- a/xtask/src/source_structure/python_source/environment.rs +++ b/xtask/src/source_structure/python_source/environment.rs @@ -6,7 +6,20 @@ use std::collections::VecDeque; use word_split::split_words; -pub(super) fn selected_utility(arguments: &[u8]) -> Option> { +/// The selected `env` utility when repository bytes determine it safely. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum UtilitySelection { + /// The exact utility word selected after options and assignments. + Known(Vec), + /// Runtime environment substitution can change the selected utility. + Ambiguous, +} + +/// Selects the utility executed by an `env` shebang. +/// +/// Invalid word framing returns absence. A selected word containing unresolved +/// `${VAR}` syntax remains explicitly ambiguous so callers can fail closed. +pub(super) fn selected_utility(arguments: &[u8]) -> Option { let mut words = VecDeque::from(split_words(arguments)?); let mut options = true; let mut split_budget = arguments.len().checked_add(1)?; @@ -40,7 +53,10 @@ pub(super) fn selected_utility(arguments: &[u8]) -> Option> { options = false; continue; } - return Some(word); + if word.windows(2).any(|window| window == b"${") { + return Some(UtilitySelection::Ambiguous); + } + return Some(UtilitySelection::Known(word)); } None } @@ -131,7 +147,7 @@ mod tests { ] { assert_eq!( super::selected_utility(arguments), - Some(b"python3".to_vec()) + Some(super::UtilitySelection::Known(b"python3".to_vec())) ); } } @@ -143,7 +159,18 @@ mod tests { b"-S sh -c 'echo python3'", b"-S \"sh -c 'echo python3'\"", ] { - assert_eq!(super::selected_utility(arguments), Some(b"sh".to_vec())); + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"sh".to_vec())) + ); } } + + #[test] + fn unresolved_selected_utility_is_ambiguous() { + assert_eq!( + super::selected_utility(b"-S '${UNSET_INTERPRETER}sh'"), + Some(super::UtilitySelection::Ambiguous) + ); + } } From 3eb4d2af8b2d8cdf79b4ceb6a6b1690a3fd15d41 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:38:01 -0700 Subject: [PATCH 066/113] Fix: make source inventory partition total --- xtask/src/source_structure.rs | 2 +- .../executable_candidate_tests.rs | 18 ++++++++- .../src/source_structure/source_inventory.rs | 38 ++++++++++++------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 3cc2047..1197525 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -63,7 +63,7 @@ fn inventory_violations( inventory: SourceInventory, ) -> Result, SourceStructureError> { for relative in inventory.executable_candidates { - let _execution = refuse_executable_python(source_root, &relative)?; + let _execution = refuse_executable_python(source_root, relative.as_path())?; } source_violations(source_root, inventory.modules) } diff --git a/xtask/src/source_structure/executable_candidate_tests.rs b/xtask/src/source_structure/executable_candidate_tests.rs index 99a6c9c..162107f 100644 --- a/xtask/src/source_structure/executable_candidate_tests.rs +++ b/xtask/src/source_structure/executable_candidate_tests.rs @@ -67,12 +67,28 @@ fn non_utf8_executable_candidate_preserves_path_bytes() -> Result<(), Box Result<(), Box> { + let present = BTreeSet::from([ + GitPath::new(b"module.rs".to_vec()), + GitPath::new(b"script.txt".to_vec()), + GitPath::new(b"removed.sh".to_vec()), + ]); + let deleted = BTreeSet::from([GitPath::new(b"removed.sh".to_vec())]); + + let inventory = super::select_source_inventory(&present, &deleted)?; + + assert_eq!(inventory.modules.len(), 1); + assert_eq!(inventory.executable_candidates.len(), 1); + Ok(()) +} + fn make_executable(path: &Path) -> Result<(), std::io::Error> { let mut permissions = fs::metadata(path)?.permissions(); permissions.set_mode(0o755); diff --git a/xtask/src/source_structure/source_inventory.rs b/xtask/src/source_structure/source_inventory.rs index 2d06483..d20bf5a 100644 --- a/xtask/src/source_structure/source_inventory.rs +++ b/xtask/src/source_structure/source_inventory.rs @@ -21,7 +21,17 @@ pub(super) const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ pub(super) struct SourceInventory { pub(super) modules: Vec, - pub(super) executable_candidates: Vec, + pub(super) executable_candidates: Vec, +} + +/// A repository-relative path admitted for executable inspection. +pub(super) struct InspectionPath(PathBuf); + +impl InspectionPath { + /// Returns the validated platform path. + pub(super) fn as_path(&self) -> &Path { + &self.0 + } } pub(super) fn collect(repository_root: &Path) -> Result { @@ -42,27 +52,27 @@ pub(super) fn select( present: &BTreeSet, deleted: &BTreeSet, ) -> Result { - let modules = select_source_paths(present, deleted)?; - let executable_candidates = present - .difference(deleted) - .filter(|path| !is_source_candidate(path.as_bytes())) - .map(admit_inspection_path) - .collect::, _>>()?; + let mut modules = Vec::new(); + let mut executable_candidates = Vec::new(); + for path in present.difference(deleted) { + if is_source_candidate(path.as_bytes()) { + modules.push(admit_source_path(path)?); + } else { + executable_candidates.push(admit_inspection_path(path)?); + } + } Ok(SourceInventory { modules, executable_candidates, }) } +#[cfg(test)] pub(super) fn select_source_paths( present: &BTreeSet, deleted: &BTreeSet, ) -> Result, SourceStructureError> { - present - .difference(deleted) - .filter(|path| is_source_candidate(path.as_bytes())) - .map(admit_source_path) - .collect() + select(present, deleted).map(|inventory| inventory.modules) } fn admit_source_path(path: &GitPath) -> Result { @@ -78,13 +88,13 @@ fn admit_source_path(path: &GitPath) -> Result Result { +fn admit_inspection_path(path: &GitPath) -> Result { let relative = PathBuf::from(OsString::from_vec(path.as_bytes().to_vec())); if relative .components() .all(|component| matches!(component, Component::Normal(_))) { - Ok(relative) + Ok(InspectionPath(relative)) } else { Err(SourceStructureError::InvalidPath(path_text( path, From d76435aa05a6dda6f14cd8237cdcd2942019208f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:40:55 -0700 Subject: [PATCH 067/113] Fix: bound every executable source file --- CHANGELOG.md | 2 +- docs/Rust Standards.md | 3 +- xtask/src/source_structure.rs | 28 ++++++++++++------- .../executable_candidate_tests.rs | 21 ++++++++++++++ xtask/src/source_structure/source_error.rs | 6 ++-- xtask/src/source_structure/tests.rs | 19 +++++++------ 6 files changed, 55 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6345a36..2057944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ after its public API and format compatibility policies are established. Rust `xtask`, cross-checks every identity-bearing digest against external `b3sum`, and CI refuses Rust, Python, or shell source modules that exceed the documented 500-physical-line hard maximum, including test modules and - extensionless executable sources. + executable sources regardless of filename suffix. - Repository source verification now uses capability-relative, no-follow file opens and verifies repository-root identity after Git inventory and again after source scanning, so a persistent root replacement or source path diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 6af4a1c..31e85d2 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -477,7 +477,8 @@ Hard CI limits: Count physical lines for deterministic enforcement. Blank and comment lines remain part of the maintainability surface; reviewers should also examine -logical structure. +logical structure. The hard maximum applies to every executable source file +regardless of its filename suffix. A file above 300 lines MUST begin with a decomposition issue or contain an approved exception explaining why splitting it would damage locality. diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 1197525..2dcc706 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -61,17 +61,25 @@ fn verify_source_root( fn inventory_violations( source_root: &RepositoryRoot, inventory: SourceInventory, -) -> Result, SourceStructureError> { +) -> Result, SourceStructureError> { + let mut violations = Vec::new(); for relative in inventory.executable_candidates { - let _execution = refuse_executable_python(source_root, relative.as_path())?; + let execution = refuse_executable_python(source_root, relative.as_path())?; + if execution == FileExecution::Executable + && source_line_count(source_root, relative.as_path())? == SourceLineCount::Exceeded + { + violations.push(relative.as_path().to_owned()); + } } - source_violations(source_root, inventory.modules) + violations.extend(source_violations(source_root, inventory.modules)?); + violations.sort(); + Ok(violations) } fn source_violations( source_root: &RepositoryRoot, paths: Vec, -) -> Result, SourceStructureError> { +) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { let execution = refuse_executable_python(source_root, relative.as_path())?; @@ -85,9 +93,9 @@ fn source_violations( { continue; } - let lines = source_line_count(source_root, &relative)?; + let lines = source_line_count(source_root, relative.as_path())?; if lines == SourceLineCount::Exceeded { - violations.push(relative.as_str().to_owned()); + violations.push(relative.as_path().to_owned()); } } Ok(violations) @@ -95,18 +103,18 @@ fn source_violations( fn source_line_count( source_root: &RepositoryRoot, - relative: &RepositoryPath, + relative: &Path, ) -> Result { source_line_count_with(source_root, relative, RepositoryRoot::open_file) } fn source_line_count_with( source_root: &RepositoryRoot, - relative: &RepositoryPath, + relative: &Path, open_source: impl FnOnce(&RepositoryRoot, &Path) -> Result, ) -> Result { - let path = source_root.display_path(relative.as_path()); - let file = open_source(source_root, relative.as_path()).map_err(|error| match error { + let path = source_root.display_path(relative); + let file = open_source(source_root, relative).map_err(|error| match error { OpenRepositoryFileError::Io(source) => SourceStructureError::Inspect { path: path.clone(), source, diff --git a/xtask/src/source_structure/executable_candidate_tests.rs b/xtask/src/source_structure/executable_candidate_tests.rs index 162107f..1e9389c 100644 --- a/xtask/src/source_structure/executable_candidate_tests.rs +++ b/xtask/src/source_structure/executable_candidate_tests.rs @@ -56,6 +56,27 @@ fn non_executable_text_remains_outside_the_python_boundary() Ok(()) } +#[test] +fn non_source_suffix_executable_obeys_the_line_limit() -> Result<(), Box> { + let directory = TestDirectory::create("hidden-source-limit")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + let script = repository.join("check.bash"); + let limit = usize::try_from(super::SOURCE_MODULE_HARD_LIMIT_LINES)?; + fs::write(&script, format!("#!/bin/sh\n{}", ":\n".repeat(limit)))?; + make_executable(&script)?; + let present = BTreeSet::from([GitPath::new(b"check.bash".to_vec())]); + + let paths = super::select_source_inventory(&present, &BTreeSet::new())?; + let source_root = RepositoryRoot::open(&repository)?; + let violations = super::inventory_violations(&source_root, paths)?; + + assert_eq!(violations, vec![Path::new("check.bash").to_owned()]); + drop(source_root); + directory.close()?; + Ok(()) +} + #[test] fn non_utf8_executable_candidate_preserves_path_bytes() -> Result<(), Box> { let path_bytes = b"check-\xff.txt"; diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 30162de..8908996 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -25,7 +25,7 @@ pub(crate) enum SourceStructureError { RepositoryRootChanged(PathBuf), Violations { maximum: u64, - paths: Vec, + paths: Vec, }, } @@ -96,7 +96,7 @@ impl From for SourceStructureError { fn violations_display( formatter: &mut fmt::Formatter<'_>, maximum: u64, - paths: &[String], + paths: &[PathBuf], ) -> fmt::Result { write!( formatter, @@ -104,7 +104,7 @@ fn violations_display( )?; for path in paths { formatter.write_str("; ")?; - escaped_controls(formatter, path)?; + escaped_path(formatter, path)?; write!(formatter, ": >{maximum}")?; } Ok(()) diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index 8966ab4..d7590a6 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -46,7 +46,7 @@ fn early_source_refusal_does_not_claim_an_exact_line_count() { let diagnostic = line_count(BufReader::new(reader)).map(|observed| match observed { SourceLineCount::Exceeded => super::SourceStructureError::Violations { maximum: 500, - paths: vec![String::from("src/large.rs")], + paths: vec![std::path::PathBuf::from("src/large.rs")], } .to_string(), SourceLineCount::Within(lines) => format!("unexpected exact count: {lines}"), @@ -78,7 +78,7 @@ fn source_structure_diagnostics_are_stable() { ); let violations = super::SourceStructureError::Violations { maximum: 7, - paths: vec![String::from("src/large.rs")], + paths: vec![std::path::PathBuf::from("src/large.rs")], }; assert_eq!( violations.to_string(), @@ -170,7 +170,7 @@ fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box Result<(), super::SourceStructur })?; let relative = RepositoryPath::admit(String::from("source.rs"))?; - let result = source_line_count_with(&source_root, &relative, |source_root, relative| { - let admitted = source_root.display_path(relative); - fs::rename(&admitted, &retained_path).map_err(OpenRepositoryFileError::Io)?; - symlink(&target_path, &admitted).map_err(OpenRepositoryFileError::Io)?; - source_root.open_file(relative) - }); + let result = + source_line_count_with(&source_root, relative.as_path(), |source_root, relative| { + let admitted = source_root.display_path(relative); + fs::rename(&admitted, &retained_path).map_err(OpenRepositoryFileError::Io)?; + symlink(&target_path, &admitted).map_err(OpenRepositoryFileError::Io)?; + source_root.open_file(relative) + }); let refused = matches!( result, Err(super::SourceStructureError::Inspect { From 371f125926a59ce356c4e47d48a36b12474b98d9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:42:18 -0700 Subject: [PATCH 068/113] Fix: require documentation workflow triggers --- CHANGELOG.md | 3 +- .../workflow_contract.rs | 15 +++++++++ .../workflow_contract/tests.rs | 32 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2057944..7640e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ after its public API and format compatibility policies are established. unreviewed workflow/job run defaults and step execution fields, pins the documentation runner and job deadline, rejects guarded or failure-tolerant documentation jobs and required steps, requires each Dependabot update block - to choose exactly one directory field form, + to choose exactly one directory field form, and requires the documentation + workflow to run for pushes to `main` and every pull request, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index b6223cd..ec6ed7e 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -58,9 +58,24 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> let [document] = documents.as_slice() else { return Err(contract("workflow contains exactly one YAML document")); }; + if !triggers_are_reviewed(document) { + return Err(contract( + "workflow runs on reviewed push and pull request triggers", + )); + } reviewed_runs(documentation_steps(document)?) } +fn triggers_are_reviewed(document: &Yaml) -> bool { + let triggers = &document["on"]; + let push = &triggers["push"]; + let branches = &push["branches"]; + mapping_has_exact_fields(triggers, &["push", "pull_request"]) + && mapping_has_exact_fields(push, &["branches"]) + && matches!(branches.as_vec().map(Vec::as_slice), Some([branch]) if branch.as_str() == Some("main")) + && triggers["pull_request"].is_null() +} + fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError> { if !document["defaults"].is_badvalue() || !document["env"].is_badvalue() { return Err(contract( diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index a74b515..6fadc86 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -7,6 +7,10 @@ mod execution_context; mod node_setup; const WORKFLOW: &str = r#"name: CI +on: + push: + branches: [main] + pull_request: jobs: documentation: name: Documentation and workflow integrity @@ -47,6 +51,30 @@ fn documentation_job_delegates_once_to_the_rust_boundary() { assert!(super::admit(WORKFLOW).is_ok()); } +#[test] +fn documentation_job_requires_reviewed_workflow_triggers() { + for workflow in [ + WORKFLOW.replace(" pull_request:\n", ""), + WORKFLOW.replacen( + "on:\n push:\n branches: [main]\n pull_request:\n", + "on:\n workflow_dispatch:\n", + 1, + ), + WORKFLOW.replace( + " pull_request:\n", + " pull_request:\n paths:\n - README.md\n", + ), + ] { + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "workflow runs on reviewed push and pull request triggers", + }) + )); + } +} + #[test] fn documentation_job_requires_the_malformed_input_regressions() { let runs = super::REVIEWED_RUNS @@ -80,6 +108,10 @@ fn documentation_job_refuses_python_execution() { #[test] fn inert_yaml_cannot_impersonate_documentation_commands() { let workflow = r#"name: CI +on: + push: + branches: [main] + pull_request: jobs: documentation: name: Documentation and workflow integrity From c896c15bf16bb21fbe7cb10ab3636c5c0c424766 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:46:56 -0700 Subject: [PATCH 069/113] Fix: preserve documentation step order --- CHANGELOG.md | 5 +- .../workflow_contract.rs | 71 ++++++++++--------- .../workflow_contract/action_steps.rs | 25 +++++++ .../workflow_contract/reviewed_step.rs | 62 ++++++++++++++++ .../workflow_contract/tests.rs | 7 +- 5 files changed, 129 insertions(+), 41 deletions(-) create mode 100644 xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7640e15..61395ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ after its public API and format compatibility policies are established. declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed documentation CI command and the pinned Node setup action exactly once, - admits only the exact pinned checkout and Node setup actions in their reviewed - order, rejects checkout overrides and unreviewed action steps, refuses + admits only the exact pinned checkout and Node setup actions, requires actions + and commands to execute in one reviewed order, rejects checkout overrides and + unreviewed action steps, refuses alternate setup-node actions, requires the reviewed Node version, rejects unreviewed workflow/job run defaults and step execution fields, pins the documentation runner and job deadline, rejects guarded or failure-tolerant diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index ec6ed7e..0d02616 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -1,11 +1,14 @@ //! This module owns the CI documentation-job execution contract. +mod reviewed_step; + use yaml_rust2::{Yaml, YamlLoader}; use crate::repository_file::RepositoryRoot; use super::error::DocumentationError; use super::repository_text; +use reviewed_step::{DocumentationStep, REVIEWED_STEPS, steps_have_reviewed_membership}; const CI_PATH: &str = ".github/workflows/ci.yml"; const CHECKOUT_ACTION: &str = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; @@ -14,23 +17,9 @@ const DOCUMENTATION_JOB_FIELDS: &[&str] = &["name", "runs-on", "timeout-minutes" const DOCUMENTATION_JOB_NAME: &str = "Documentation and workflow integrity"; const DOCUMENTATION_RUNNER: &str = "ubuntu-latest"; const DOCUMENTATION_TIMEOUT_MINUTES: i64 = 10; -const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ - documentation_integrity::execution::external_tests -- --ignored"; const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020"; const SETUP_NODE_ACTION_PREFIX: &str = "actions/setup-node@"; const NODE_VERSION: &str = "24.18.0"; -const XTASK_COMMAND: &str = "cargo xtask documentation-integrity-check"; -const REVIEWED_RUNS: &[&str] = &[ - "rustup show", - r#"documentation_tools="$RUNNER_TEMP/documentation-tools" -scripts/install_documentation_tools.sh "$documentation_tools" -printf '%s\n' \ - "$documentation_tools/bin" \ - "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH""#, - MALFORMED_INPUT_COMMAND, - XTASK_COMMAND, - r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#, -]; pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { let workflow = repository_text::read(repository_root, CI_PATH)?; @@ -38,17 +27,23 @@ pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), Documentatio } fn admit(workflow: &str) -> Result<(), DocumentationError> { - let runs = documentation_runs(workflow)?; - if !runs_are_reviewed(&runs) { - return Err(contract(concat!( + let steps = admitted_steps(workflow)?; + if steps.as_slice() == REVIEWED_STEPS { + return Ok(()); + } + if steps_have_reviewed_membership(&steps) { + Err(contract( + "documentation job steps execute in reviewed order", + )) + } else { + Err(contract(concat!( "documentation job run commands are reviewed and required ", "commands execute once" - ))); + ))) } - Ok(()) } -fn documentation_runs(workflow: &str) -> Result, DocumentationError> { +fn admitted_steps(workflow: &str) -> Result, DocumentationError> { let documents = YamlLoader::load_from_str(workflow).map_err(|source| { DocumentationError::RepositoryYaml { path: CI_PATH, @@ -63,7 +58,7 @@ fn documentation_runs(workflow: &str) -> Result, DocumentationError> "workflow runs on reviewed push and pull request triggers", )); } - reviewed_runs(documentation_steps(document)?) + reviewed_steps(documentation_steps(document)?) } fn triggers_are_reviewed(document: &Yaml) -> bool { @@ -107,12 +102,19 @@ fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError Ok(steps) } -fn reviewed_runs(steps: &[Yaml]) -> Result, DocumentationError> { - let mut runs = Vec::new(); +fn reviewed_steps(steps: &[Yaml]) -> Result, DocumentationError> { + let mut admitted = Vec::new(); let mut actions = Vec::new(); for step in steps { - actions.extend(admit_action(step)?); - runs.extend(admit_run(step)?); + if let Some(action) = admit_action(step)? { + actions.push(action); + admitted.push(match action { + DocumentationAction::Checkout => DocumentationStep::Checkout, + DocumentationAction::Node => DocumentationStep::Node, + }); + } else if let Some(run) = admit_run(step)? { + admitted.push(run); + } } if actions .iter() @@ -129,7 +131,7 @@ fn reviewed_runs(steps: &[Yaml]) -> Result, DocumentationError> { "documentation job actions execute in reviewed order", )); } - Ok(runs) + Ok(admitted) } fn admit_action(step: &Yaml) -> Result, DocumentationError> { @@ -207,7 +209,7 @@ fn admit_action_execution( Ok(()) } -fn admit_run(step: &Yaml) -> Result, DocumentationError> { +fn admit_run(step: &Yaml) -> Result, DocumentationError> { let run = &step["run"]; if run.is_badvalue() { return Ok(None); @@ -226,14 +228,13 @@ fn admit_run(step: &Yaml) -> Result, DocumentationError> { let Some(run) = run.as_str() else { return Err(contract("documentation job run values are strings")); }; - Ok(Some(run.trim_end_matches('\n').to_owned())) -} - -fn runs_are_reviewed(runs: &[String]) -> bool { - runs.len() == REVIEWED_RUNS.len() - && REVIEWED_RUNS - .iter() - .all(|required| runs.iter().filter(|run| run.as_str() == *required).count() == 1) + let Some(admitted) = DocumentationStep::from_run(run.trim_end_matches('\n')) else { + return Err(contract(concat!( + "documentation job run commands are reviewed and required ", + "commands execute once" + ))); + }; + Ok(Some(admitted)) } fn mapping_has_exact_fields(mapping: &Yaml, fields: &[&str]) -> bool { diff --git a/xtask/src/documentation_integrity/workflow_contract/action_steps.rs b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs index 077cc2c..1e75284 100644 --- a/xtask/src/documentation_integrity/workflow_contract/action_steps.rs +++ b/xtask/src/documentation_integrity/workflow_contract/action_steps.rs @@ -15,6 +15,15 @@ const NODE_STEP: &str = concat!( " with:\n", " node-version: 24.18.0\n" ); +const INSTALL_STEP: &str = concat!( + " - name: Install documentation tools\n", + " run: |\n", + " documentation_tools=\"$RUNNER_TEMP/documentation-tools\"\n", + " scripts/install_documentation_tools.sh \"$documentation_tools\"\n", + " printf '%s\\n' \\\n", + " \"$documentation_tools/bin\" \\\n", + " \"$documentation_tools/npm/node_modules/.bin\" >> \"$GITHUB_PATH\"\n" +); #[test] fn checkout_of_an_unreviewed_revision_does_not_satisfy_the_contract() { @@ -79,3 +88,19 @@ fn documentation_actions_execute_in_reviewed_order() { }) )); } + +#[test] +fn actions_and_commands_execute_in_one_reviewed_order() { + let moved = format!("{INSTALL_STEP}{NODE_STEP}"); + let workflow = WORKFLOW + .replace(NODE_STEP, "") + .replace(INSTALL_STEP, &moved); + + assert!(matches!( + admit(&workflow), + Err(DocumentationError::RepositoryContract { + path: CI_PATH, + requirement: "documentation job steps execute in reviewed order", + }) + )); +} diff --git a/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs new file mode 100644 index 0000000..2199fad --- /dev/null +++ b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs @@ -0,0 +1,62 @@ +//! This module owns the exact documentation-job step sequence. + +const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ + documentation_integrity::execution::external_tests -- --ignored"; +const INSTALL_TOOLS_COMMAND: &str = r#"documentation_tools="$RUNNER_TEMP/documentation-tools" +scripts/install_documentation_tools.sh "$documentation_tools" +printf '%s\n' \ + "$documentation_tools/bin" \ + "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH""#; +const DIFF_CHECK_COMMAND: &str = r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#; + +/// The only admitted documentation-job execution sequence. +pub(super) const REVIEWED_STEPS: &[DocumentationStep] = &[ + DocumentationStep::Checkout, + DocumentationStep::Rustup, + DocumentationStep::Node, + DocumentationStep::InstallTools, + DocumentationStep::MalformedInputs, + DocumentationStep::Verify, + DocumentationStep::DiffCheck, +]; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// One semantically reviewed documentation-job step. +pub(super) enum DocumentationStep { + /// Opens the exact repository revision without retained credentials. + Checkout, + /// Confirms the pinned Rust toolchain selected by repository policy. + Rustup, + /// Installs the pinned Node.js runtime. + Node, + /// Installs the lockfile-bound documentation tools. + InstallTools, + /// Runs malformed-input regression evidence. + MalformedInputs, + /// Runs the Rust documentation-integrity boundary. + Verify, + /// Refuses whitespace errors across the reviewed tree. + DiffCheck, +} + +impl DocumentationStep { + /// Classifies one exact reviewed `run` body. + pub(super) fn from_run(run: &str) -> Option { + match run { + "rustup show" => Some(Self::Rustup), + INSTALL_TOOLS_COMMAND => Some(Self::InstallTools), + MALFORMED_INPUT_COMMAND => Some(Self::MalformedInputs), + "cargo xtask documentation-integrity-check" => Some(Self::Verify), + DIFF_CHECK_COMMAND => Some(Self::DiffCheck), + _ => None, + } + } +} + +/// Reports whether every reviewed step occurs exactly once. +pub(super) fn steps_have_reviewed_membership(steps: &[DocumentationStep]) -> bool { + steps.len() == REVIEWED_STEPS.len() + && REVIEWED_STEPS + .iter() + .all(|required| steps.iter().filter(|step| *step == required).count() == 1) +} diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 6fadc86..8666822 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -77,14 +77,13 @@ fn documentation_job_requires_reviewed_workflow_triggers() { #[test] fn documentation_job_requires_the_malformed_input_regressions() { - let runs = super::REVIEWED_RUNS + let steps = super::REVIEWED_STEPS .iter() .copied() - .filter(|run| *run != super::MALFORMED_INPUT_COMMAND) - .map(String::from) + .filter(|step| *step != super::DocumentationStep::MalformedInputs) .collect::>(); - assert!(!super::runs_are_reviewed(&runs)); + assert!(!super::steps_have_reviewed_membership(&steps)); } #[test] From 0454bf908bf1f40c821684fb9f65ca9dd29ebc7f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:48:49 -0700 Subject: [PATCH 070/113] Fix: normalize no-follow symlink refusal --- CHANGELOG.md | 7 ++++--- xtask/src/documentation_integrity/corpus.rs | 8 -------- .../src/documentation_integrity/corpus/tests.rs | 6 +++++- xtask/src/repository_file.rs | 17 ++++++++++++++--- xtask/src/source_structure/tests.rs | 5 +---- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61395ce..4747600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,9 +65,10 @@ after its public API and format compatibility policies are established. documented 500-physical-line hard maximum, including test modules and executable sources regardless of filename suffix. - Repository source verification now uses capability-relative, no-follow file - opens and verifies repository-root identity after Git inventory and again - after source scanning, so a persistent root replacement or source path - replaced with a symlink is refused. The pure Rust boundary also refuses + opens, normalizes symlink refusal at that capability boundary across Unix + error conventions, and verifies repository-root identity after Git inventory + and again after source scanning, so a persistent root replacement or source + path replaced with a symlink is refused. The pure Rust boundary also refuses `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every executable regular file regardless of filename suffix, including raw non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index 80eafcf..8312264 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -162,14 +162,6 @@ fn admit_path( Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { Ok(None) } - Err(OpenRepositoryFileError::Io(source)) - if source.raw_os_error() == Some(rustix::io::Errno::LOOP.raw_os_error()) => - { - Err(DocumentationError::NonRegular { - corpus: kind.label(), - path: text, - }) - } Err(OpenRepositoryFileError::Io(source)) => Err(DocumentationError::Inspect { corpus: kind.label(), path: text, diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs index b1e757c..64ebe30 100644 --- a/xtask/src/documentation_integrity/corpus/tests.rs +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -7,7 +7,7 @@ use std::process::Command; use super::{CorpusKind, SourceCorpus, admit_path}; use crate::documentation_integrity::error::DocumentationError; use crate::git_inventory::GitPath; -use crate::repository_file::RepositoryRoot; +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; use crate::test_directory::TestDirectory; #[test] @@ -134,6 +134,10 @@ fn symlinked_markdown_is_refused() -> Result<(), Box> { symlink("target.txt", root.join("linked.md"))?; let repository_root = RepositoryRoot::open(root)?; + assert!(matches!( + repository_root.open_file(Path::new("linked.md")), + Err(OpenRepositoryFileError::NonRegular) + )); let process_directory = repository_root.process_directory()?; let result = SourceCorpus::markdown(&repository_root, &process_directory); diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs index 61ceb62..4ba135d 100644 --- a/xtask/src/repository_file.rs +++ b/xtask/src/repository_file.rs @@ -90,11 +90,22 @@ impl RepositoryRoot { } pub(crate) fn open_file(&self, relative: &Path) -> Result { - let file = self + let file = match self .directory .open_with(relative, &REPOSITORY_READ_POLICY.options()) - .map_err(OpenRepositoryFileError::Io)? - .into_std(); + { + Ok(file) => file.into_std(), + Err(source) => { + if self + .directory + .symlink_metadata(relative) + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(OpenRepositoryFileError::NonRegular); + } + return Err(OpenRepositoryFileError::Io(source)); + } + }; let metadata = file.metadata().map_err(OpenRepositoryFileError::Io)?; if metadata.is_file() { Ok(file) diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index d7590a6..a9deda9 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -253,10 +253,7 @@ fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructur }); let refused = matches!( result, - Err(super::SourceStructureError::Inspect { - ref path, - source: _, - }) if path == &source_path + Err(super::SourceStructureError::NonRegular(ref path)) if path == &source_path ); assert!(refused); drop(source_root); From 44127c27b016b4696c8cee421399af7b5fc3c7ba Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 17:53:53 -0700 Subject: [PATCH 071/113] Fix: bound documentation corpus bytes --- CHANGELOG.md | 2 + docs/Documentation Standards.md | 4 +- xtask/src/documentation_integrity/corpus.rs | 39 ++++++++++--- .../corpus/byte_budget.rs | 46 +++++++++++++++ .../documentation_integrity/corpus/tests.rs | 56 +++++++++++++++++++ xtask/src/documentation_integrity/error.rs | 17 +++++- .../documentation_integrity/error/display.rs | 20 +++++++ 7 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 xtask/src/documentation_integrity/corpus/byte_budget.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4747600..3820817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ after its public API and format compatibility policies are established. documentation jobs and required steps, requires each Dependabot update block to choose exactly one directory field form, and requires the documentation workflow to run for pushes to `main` and every pull request, + bounds each admitted documentation source to 4 MiB and each selected corpus + to 64 MiB before external tools start, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index 21d618b..4a3afea 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -544,7 +544,9 @@ deliberate rule choices. The Rust checker selects tracked Markdown plus nonignored new Markdown, disables configuration globs for that invocation, and refuses a different tool version. It also runs `lychee` 0.21.0 offline with fragment checking, so external-site availability cannot affect the -result. Run it from the repository root. The two Git commands inspect +result. Before external tools start, corpus admission refuses any source over +4 MiB and any selected corpus over 64 MiB. Run it from the repository root. +The two Git commands inspect unstaged and staged whitespace errors separately. The same Rust command checks workflows with `actionlint` 1.7.12 and refuses diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index 8312264..e78f042 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -1,5 +1,7 @@ //! This module owns deterministic documentation source selection. +mod byte_budget; + use std::io; use xtask::protocol_admission::posix_relative_path; @@ -7,6 +9,9 @@ use xtask::protocol_admission::posix_relative_path; use super::error::DocumentationError; use crate::git_inventory::{GitPath, paths_with}; use crate::repository_file::{OpenRepositoryFileError, RepositoryProcessDirectory, RepositoryRoot}; +use byte_budget::CorpusByteBudget; +#[cfg(test)] +use byte_budget::{CORPUS_FILE_MAX_BYTES, CORPUS_MAX_BYTES}; const MARKDOWN_PRESENT: [&str; 7] = [ "ls-files", @@ -41,6 +46,10 @@ pub(super) struct SourceCorpus { paths: Vec, } +struct AdmittedSource { + bytes: u64, + path: String, +} #[derive(Clone, Copy)] enum CorpusKind { Markdown, @@ -133,20 +142,22 @@ fn admit_paths<'a>( paths: impl Iterator, kind: CorpusKind, ) -> Result, DocumentationError> { - paths - .filter_map(|path| match admit_path(repository_root, path, kind) { - Ok(Some(path)) => Some(Ok(path)), - Ok(None) => None, - Err(error) => Some(Err(error)), - }) - .collect() + let mut admitted = Vec::new(); + let mut budget = CorpusByteBudget::default(); + for path in paths { + if let Some(source) = admit_path(repository_root, path, kind)? { + budget.admit(kind, &source)?; + admitted.push(source.path); + } + } + Ok(admitted) } fn admit_path( repository_root: &RepositoryRoot, path: &GitPath, kind: CorpusKind, -) -> Result, DocumentationError> { +) -> Result, DocumentationError> { let text = String::from_utf8(path.as_bytes().to_vec()).map_err(|source| { DocumentationError::PathEncoding { corpus: kind.label(), @@ -158,7 +169,17 @@ fn admit_path( path: text.clone(), })?; match repository_root.open_file(&relative) { - Ok(_file) => Ok(Some(text)), + Ok(file) => { + let bytes = file + .metadata() + .map_err(|source| DocumentationError::Inspect { + corpus: kind.label(), + path: text.clone(), + source, + })? + .len(); + Ok(Some(AdmittedSource { bytes, path: text })) + } Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { Ok(None) } diff --git a/xtask/src/documentation_integrity/corpus/byte_budget.rs b/xtask/src/documentation_integrity/corpus/byte_budget.rs new file mode 100644 index 0000000..a3945be --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/byte_budget.rs @@ -0,0 +1,46 @@ +//! This module owns documentation corpus byte-budget admission. + +use super::{AdmittedSource, CorpusKind}; +use crate::documentation_integrity::error::DocumentationError; + +/// Maximum admitted bytes for one documentation source. +pub(super) const CORPUS_FILE_MAX_BYTES: u64 = 4_194_304; +/// Maximum admitted bytes for one complete documentation corpus. +pub(super) const CORPUS_MAX_BYTES: u64 = 67_108_864; + +/// Checked aggregate byte accounting for one corpus. +#[derive(Default)] +pub(super) struct CorpusByteBudget { + observed: u64, +} + +impl CorpusByteBudget { + /// Admits one bounded source and advances the aggregate count. + pub(super) fn admit( + &mut self, + kind: CorpusKind, + source: &AdmittedSource, + ) -> Result<(), DocumentationError> { + if source.bytes > CORPUS_FILE_MAX_BYTES { + return Err(DocumentationError::CorpusFileTooLarge { + corpus: kind.label(), + path: source.path.clone(), + maximum: CORPUS_FILE_MAX_BYTES, + observed: source.bytes, + }); + } + let observed = self + .observed + .checked_add(source.bytes) + .ok_or_else(|| DocumentationError::CorpusSizeOverflow(kind.label()))?; + if observed > CORPUS_MAX_BYTES { + return Err(DocumentationError::CorpusTooLarge { + corpus: kind.label(), + maximum: CORPUS_MAX_BYTES, + observed, + }); + } + self.observed = observed; + Ok(()) + } +} diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs index 64ebe30..68a8221 100644 --- a/xtask/src/documentation_integrity/corpus/tests.rs +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -1,5 +1,6 @@ //! This module owns documentation source-corpus regression evidence. +use std::collections::BTreeSet; use std::fs; use std::path::Path; use std::process::Command; @@ -152,6 +153,61 @@ fn symlinked_markdown_is_refused() -> Result<(), Box> { Ok(()) } +#[test] +fn oversized_markdown_is_refused_before_external_validation() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-markdown-size")?; + let root = directory.path(); + let path = root.join("oversized.md"); + let observed = 4_194_304_u64 + .checked_add(1) + .ok_or("corpus file bound overflow")?; + fs::File::create(&path)?.set_len(observed)?; + let repository_root = RepositoryRoot::open(root)?; + let git_path = GitPath::new(b"oversized.md".to_vec()); + + let paths = BTreeSet::from([git_path]); + let result = super::admit_paths(&repository_root, paths.iter(), CorpusKind::Markdown); + + assert!(matches!( + result, + Err(DocumentationError::CorpusFileTooLarge { + corpus: "Markdown", + ref path, + maximum: super::CORPUS_FILE_MAX_BYTES, + observed: actual, + }) if path == "oversized.md" && actual == observed + )); + directory.close()?; + Ok(()) +} + +#[test] +fn aggregate_documentation_bytes_are_bounded_with_checked_accounting() +-> Result<(), Box> { + let mut budget = super::CorpusByteBudget::default(); + let source = super::AdmittedSource { + bytes: super::CORPUS_FILE_MAX_BYTES, + path: String::from("bounded.md"), + }; + for _ in 0..16 { + budget.admit(CorpusKind::Markdown, &source)?; + } + let observed = super::CORPUS_MAX_BYTES + .checked_add(super::CORPUS_FILE_MAX_BYTES) + .ok_or("aggregate corpus bound overflow")?; + + assert!(matches!( + budget.admit(CorpusKind::Markdown, &source), + Err(DocumentationError::CorpusTooLarge { + corpus: "Markdown", + maximum: super::CORPUS_MAX_BYTES, + observed: actual, + }) if actual == observed + )); + Ok(()) +} + #[cfg(unix)] #[test] fn fifo_workflow_is_refused() -> Result<(), Box> { diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 6542a00..ff5773d 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -16,6 +16,18 @@ pub(crate) enum DocumentationError { first: Box, second: Box, }, + CorpusFileTooLarge { + corpus: &'static str, + path: String, + maximum: u64, + observed: u64, + }, + CorpusSizeOverflow(&'static str), + CorpusTooLarge { + corpus: &'static str, + maximum: u64, + observed: u64, + }, EmptyCorpus(&'static str), GitInventory(GitInventoryError), Inspect { @@ -123,7 +135,10 @@ impl Error for DocumentationError { Self::RepositoryRootInspect { source, .. } => Some(source), Self::ToolOutputEncoding { source, .. } => Some(source), Self::ToolUnavailable { source, .. } => Some(source), - Self::EmptyCorpus(_) + Self::CorpusFileTooLarge { .. } + | Self::CorpusSizeOverflow(_) + | Self::CorpusTooLarge { .. } + | Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } | Self::RepositoryFileNonRegular(_) diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index f98f05a..b0caf2c 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -25,6 +25,26 @@ impl fmt::Display for DocumentationError { Self::CheckFailures { first, second } => { write!(formatter, "{first}; additionally: {second}") } + Self::CorpusFileTooLarge { + corpus, + path, + maximum, + observed, + } => write!( + formatter, + "{corpus} source `{path}` is {observed} bytes; maximum is {maximum}" + ), + Self::CorpusSizeOverflow(corpus) => { + write!(formatter, "{corpus} corpus byte count overflowed") + } + Self::CorpusTooLarge { + corpus, + maximum, + observed, + } => write!( + formatter, + "{corpus} corpus is {observed} bytes; maximum is {maximum}" + ), Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), Self::GitInventory(error) => write!(formatter, "{error}"), Self::Inspect { corpus, path, .. } => { From 553519695ef55c6f714ec01dcb65fba62089bf1c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:01:45 -0700 Subject: [PATCH 072/113] Fix: fuzz repository JSON admission --- CHANGELOG.md | 3 + fuzz/Cargo.lock | 99 +++++++++++++++++++++++++ fuzz/Cargo.toml | 9 ++- fuzz/README.md | 6 ++ fuzz/fuzz_targets/repository_json.rs | 10 +++ xtask/Cargo.toml | 4 +- xtask/src/fuzz_campaign/target/tests.rs | 1 + xtask/src/lib.rs | 39 ++++++++++ xtask/tests/repository_json_fuzz.rs | 36 +++++++++ 9 files changed, 204 insertions(+), 3 deletions(-) create mode 100644 fuzz/fuzz_targets/repository_json.rs create mode 100644 xtask/tests/repository_json_fuzz.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3820817..caa5728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,9 @@ after its public API and format compatibility policies are established. - Golden protocol framing, field, hexadecimal, path, mutation-operation, and fixed-width value decoders now share a bounded fuzz surface backed by precise table-driven malformed-corpus refusals. +- Duplicate-refusing repository JSON admission now has a one-mebibyte fuzz + boundary with deterministic evidence for valid nested input, malformed JSON, + excessive nesting, and duplicate members at nested object depth. - Deterministic fuzz seed materialization now uses a capability-bound Rust `xtask`, syncs and atomically publishes derived seed files without mutating hard-link targets, recovers interrupted fixed-name staging files, cleans diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 0b03751..7d8e423 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -84,6 +84,12 @@ dependencies = [ "r-efi", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "jobserver" version = "0.1.35" @@ -126,18 +132,111 @@ dependencies = [ "cc", ] +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + [[package]] name = "xtask" version = "0.0.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index b708353..bb29289 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -12,7 +12,7 @@ cargo-fuzz = true [dependencies] keep = { path = "..", version = "=0.0.0" } libfuzzer-sys = { version = "=0.4.13", default-features = false, features = ["link_libfuzzer"] } -xtask = { path = "../xtask", version = "=0.0.0", default-features = false, features = ["golden-protocol-fuzz"] } +xtask = { path = "../xtask", version = "=0.0.0", default-features = false, features = ["golden-protocol-fuzz", "repository-json-fuzz"] } [lints.rust] warnings = "deny" @@ -110,6 +110,13 @@ test = false doc = false bench = false +[[bin]] +name = "repository_json" +path = "fuzz_targets/repository_json.rs" +test = false +doc = false +bench = false + [[bin]] name = "segment_format" path = "fuzz_targets/segment_format.rs" diff --git a/fuzz/README.md b/fuzz/README.md index 3963786..4cb71eb 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -49,6 +49,12 @@ schemas plus canonical case, canonical decimal, invalid-identity classification, and mutation parsing under the campaign's one-mebibyte input bound. +The `repository_json` target feeds arbitrary bytes through the same +duplicate-refusing parser used for the documentation-tool manifest and lock +graph. Its facade refuses inputs over one mebibyte before parsing; integration +laws cover valid nested JSON, malformed bytes, excessive nesting, and duplicate +members below an enclosing object and array. + The `layout_record` seeds are the four exact canonical binary records derived from the reviewed hexadecimal fixtures. The target feeds arbitrary bytes through the bounded decoder and, for every admitted record, requires exact diff --git a/fuzz/fuzz_targets/repository_json.rs b/fuzz/fuzz_targets/repository_json.rs new file mode 100644 index 0000000..ed85dfe --- /dev/null +++ b/fuzz/fuzz_targets/repository_json.rs @@ -0,0 +1,10 @@ +#![no_main] + +//! This target owns bounded duplicate-refusing repository JSON fuzzing. + +use libfuzzer_sys::fuzz_target; +use xtask::admit_repository_json; + +fuzz_target!(|bytes: &[u8]| { + let _ = admit_repository_json(bytes); +}); diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 7f636da..fee2de4 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -9,6 +9,7 @@ publish = false [features] default = ["repository-tasks"] golden-protocol-fuzz = [] +repository-json-fuzz = ["dep:serde", "dep:serde_json"] repository-tasks = [ "dep:blake3", "dep:cap-fs-ext", @@ -16,8 +17,7 @@ repository-tasks = [ "dep:md-5", "dep:repository-process-spawn", "dep:rustix", - "dep:serde", - "dep:serde_json", + "repository-json-fuzz", "dep:signal-hook", "dep:yaml-rust2", ] diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index f88ebb1..03518e1 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -29,6 +29,7 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "fast_cdc", "golden_protocol", "layout_record", + "repository_json", "segment_format", ] ); diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs index e0d3ddd..dfd4e8c 100644 --- a/xtask/src/lib.rs +++ b/xtask/src/lib.rs @@ -16,6 +16,14 @@ mod diagnostic; )] mod golden_protocol_fuzz; +#[cfg(feature = "repository-json-fuzz")] +#[allow( + clippy::redundant_pub_crate, + reason = "the library facade deliberately hides the repository parser implementation" +)] +#[path = "documentation_integrity/node_toolchain/unique_json.rs"] +mod repository_json; + pub mod protocol_admission; #[cfg(test)] @@ -50,3 +58,34 @@ pub fn admit_golden_protocol(selector: u8, input: &[u8]) -> GoldenProtocolAdmiss GoldenProtocolAdmission::Refused } } + +/// Whether bounded duplicate-refusing repository JSON admission accepted input. +#[cfg(feature = "repository-json-fuzz")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RepositoryJsonAdmission { + /// UTF-8 JSON within the byte and nesting limits was admitted. + Admitted, + /// Encoding, syntax, size, nesting, or duplicate members were refused. + Refused, +} + +/// Exercises the exact duplicate-refusing repository JSON parser. +/// +/// Input above one mebibyte is refused before UTF-8 decoding or recursive +/// parsing. No serializer-owned value escapes this fuzz-only boundary. +#[cfg(feature = "repository-json-fuzz")] +#[must_use] +pub fn admit_repository_json(input: &[u8]) -> RepositoryJsonAdmission { + const MAXIMUM_BYTES: usize = 1_048_576; + if input.len() > MAXIMUM_BYTES { + return RepositoryJsonAdmission::Refused; + } + let Ok(raw) = std::str::from_utf8(input) else { + return RepositoryJsonAdmission::Refused; + }; + if repository_json::parse(raw).is_ok() { + RepositoryJsonAdmission::Admitted + } else { + RepositoryJsonAdmission::Refused + } +} diff --git a/xtask/tests/repository_json_fuzz.rs b/xtask/tests/repository_json_fuzz.rs new file mode 100644 index 0000000..5894fd2 --- /dev/null +++ b/xtask/tests/repository_json_fuzz.rs @@ -0,0 +1,36 @@ +//! Integration laws for duplicate-refusing repository JSON fuzz admission. + +#![cfg(feature = "repository-json-fuzz")] + +use xtask::{RepositoryJsonAdmission, admit_repository_json}; + +#[test] +fn repository_json_fuzz_boundary_reaches_success_and_refusal_classes() { + let valid = br#"{"outer":[true,null,{"key":"value"}]}"#; + let malformed = br#"{"outer":["#; + let nested_duplicate = br#"{"outer":[{"key":1,"key":2}]}"#; + + assert_eq!( + admit_repository_json(valid), + RepositoryJsonAdmission::Admitted + ); + for refused in [malformed.as_slice(), nested_duplicate.as_slice()] { + assert_eq!( + admit_repository_json(refused), + RepositoryJsonAdmission::Refused + ); + } +} + +#[test] +fn repository_json_fuzz_boundary_refuses_resource_limits() { + let oversized = vec![b' '; 1_048_577]; + let deeply_nested = format!("{}null{}", "[".repeat(256), "]".repeat(256)); + + for refused in [oversized.as_slice(), deeply_nested.as_bytes()] { + assert_eq!( + admit_repository_json(refused), + RepositoryJsonAdmission::Refused + ); + } +} From 75ab406c0507d0e8fd58823f962d8c20affc9004 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:17:33 -0700 Subject: [PATCH 073/113] Fix: bind corpus identities through validation --- CHANGELOG.md | 4 +- ...escriptor-bound-child-working-directory.md | 15 ++ docs/dependencies/documentation-toolchain.md | 7 + xtask/src/documentation_integrity.rs | 2 +- xtask/src/documentation_integrity/corpus.rs | 129 +++++------------- .../corpus/byte_budget.rs | 13 +- .../corpus/replacement_tests.rs | 36 +++++ .../corpus/selection.rs | 73 ++++++++++ .../corpus/source_witness.rs | 112 +++++++++++++++ .../corpus/test_repository.rs | 23 ++++ .../documentation_integrity/corpus/tests.rs | 44 +++--- xtask/src/documentation_integrity/error.rs | 5 + .../documentation_integrity/error/display.rs | 62 ++++++--- .../src/documentation_integrity/execution.rs | 20 +-- .../execution/corpus_guard.rs | 51 +++++++ .../execution/tests.rs | 69 ++++++++++ 16 files changed, 509 insertions(+), 156 deletions(-) create mode 100644 xtask/src/documentation_integrity/corpus/replacement_tests.rs create mode 100644 xtask/src/documentation_integrity/corpus/selection.rs create mode 100644 xtask/src/documentation_integrity/corpus/source_witness.rs create mode 100644 xtask/src/documentation_integrity/corpus/test_repository.rs create mode 100644 xtask/src/documentation_integrity/execution/corpus_guard.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index caa5728..34cb432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,9 @@ after its public API and format compatibility policies are established. to choose exactly one directory field form, and requires the documentation workflow to run for pushes to `main` and every pull request, bounds each admitted documentation source to 4 MiB and each selected corpus - to 64 MiB before external tools start, + to 64 MiB before external tools start, retains every selected source + identity, refuses device, inode, size, modification-time, or change-time + drift before and after each external tool, and applies one deadline across captured and inherited child execution and output collection. Git-backed process fixtures ignore system and global Git configuration and diff --git a/docs/adr/0006-descriptor-bound-child-working-directory.md b/docs/adr/0006-descriptor-bound-child-working-directory.md index a550d5e..377aa16 100644 --- a/docs/adr/0006-descriptor-bound-child-working-directory.md +++ b/docs/adr/0006-descriptor-bound-child-working-directory.md @@ -14,6 +14,10 @@ its ambient pathname to a child creates another replacement window. An attacker could move the admitted directory, substitute another repository while the tools run, and restore the original before the final identity check. +Retaining the directory alone does not bind a selected corpus path to the file +admitted at that path. A source can be renamed, replaced while a tool reads it, +and restored without changing the retained repository directory identity. + The standard library accepts only a pathname for `std::process::Command::current_dir`. On macOS, an open directory exposed through `/dev/fd` cannot be used as that pathname. Changing the parent process @@ -32,6 +36,10 @@ crate. The crate admits one operation: 1. Own a close-on-exec duplicate of the admitted repository directory. 2. Register a child setup hook that calls only Rustix `fchdir`. 3. Spawn through the existing bounded process adapters. +4. Retain every selected source's device, inode, size, modification time, and + change time. +5. Reopen every selected path through the repository capability and compare + the admitted identity with the current path before and after every tool. POSIX specifies `fchdir` as async-signal-safe. The hook performs no allocation, locking, buffered I/O, ambient path lookup, or user callback. The descriptor @@ -45,6 +53,8 @@ network, or application policy. - Rechecking the repository pathname before and after tool execution does not detect a transient substitution. +- Reopening selected source paths without retaining and comparing the admitted + identity cannot distinguish the selected file from a later replacement. - `/dev/fd/` and `/proc/self/fd/` are not a portable child working directory. The former is not traversable as a directory on macOS, and the latter is not available there. @@ -61,6 +71,11 @@ Git inventory and documentation tools start in the exact opened repository even if its pathname is replaced. Parent process state remains unchanged, so parallel tests and readers are deterministic. +External tool output is admitted only while every selected path still has the +admitted device, inode, size, modification time, and change time +unchanged. Persistent replacement, in-place mutation, and +substitute-then-restore changes produce a typed corpus refusal. + The boundary is Unix-specific and deliberately narrow. Any additional unsafe operation, child hook, captured state, or consumer requires a new decision and new executable evidence. The crate's regression test replaces the ambient diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index d1e07c4..b84db5e 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -58,6 +58,13 @@ the configured repository path, running checks against a substitute, and restoring the original path cannot redirect either corpus selection or validation. +Each selected source also retains its device, inode, size, modification time, +and change time. The Rust boundary reopens every path through the retained +repository capability and compares that identity before and after each +external tool. A source replacement, in-place mutation, or +substitute-then-restore sequence refuses the corpus instead of admitting tool +output from ambiguous bytes. + The workflow checker disables `actionlint`'s optional `shellcheck` and `pyflakes` integrations. Neither auxiliary executable is admitted or pinned by this toolchain, so ambient PATH contents cannot expand the validation boundary. diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 664949f..508aa2d 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -36,7 +36,7 @@ pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { workflow_contract::check(&repository_root)?; let markdown = corpus::SourceCorpus::markdown(&repository_root, &process_directory)?; let workflows = corpus::SourceCorpus::workflow(&repository_root, &process_directory)?; - execution::run(&process_directory, markdown.paths(), workflows.paths())?; + execution::run(&process_directory, &repository_root, &markdown, &workflows)?; verify_root(&repository_root, repository_path) } diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index e78f042..638ee80 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -1,6 +1,10 @@ //! This module owns deterministic documentation source selection. mod byte_budget; +mod selection; +mod source_witness; +#[cfg(test)] +pub(super) mod test_repository; use std::io; @@ -12,48 +16,13 @@ use crate::repository_file::{OpenRepositoryFileError, RepositoryProcessDirectory use byte_budget::CorpusByteBudget; #[cfg(test)] use byte_budget::{CORPUS_FILE_MAX_BYTES, CORPUS_MAX_BYTES}; - -const MARKDOWN_PRESENT: [&str; 7] = [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - "*.md", -]; -const MARKDOWN_DELETED: [&str; 5] = ["ls-files", "-z", "--deleted", "--", "*.md"]; -const WORKFLOW_PRESENT: [&str; 8] = [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", - "--", - ".github/workflows/*.yml", - ".github/workflows/*.yaml", -]; -const WORKFLOW_DELETED: [&str; 6] = [ - "ls-files", - "-z", - "--deleted", - "--", - ".github/workflows/*.yml", - ".github/workflows/*.yaml", -]; +use selection::CorpusKind; +use source_witness::AdmittedSource; pub(super) struct SourceCorpus { + kind: CorpusKind, paths: Vec, -} - -struct AdmittedSource { - bytes: u64, - path: String, -} -#[derive(Clone, Copy)] -enum CorpusKind { - Markdown, - Workflow, + sources: Vec, } impl SourceCorpus { @@ -75,6 +44,16 @@ impl SourceCorpus { &self.paths } + pub(super) fn verify_unchanged( + &self, + repository_root: &RepositoryRoot, + ) -> Result<(), DocumentationError> { + for source in &self.sources { + source.verify(repository_root, self.kind)?; + } + Ok(()) + } + fn read( repository_root: &RepositoryRoot, process_directory: &RepositoryProcessDirectory, @@ -91,48 +70,19 @@ impl SourceCorpus { |command| process_directory.spawn(command), )?; let selected = present.difference(&deleted); - let paths = admit_paths(repository_root, selected, kind)?; - if paths.is_empty() { + let sources = admit_paths(repository_root, selected, kind)?; + if sources.is_empty() { Err(DocumentationError::EmptyCorpus(kind.label())) } else { - Ok(Self { paths }) - } - } -} - -impl CorpusKind { - const fn label(self) -> &'static str { - match self { - Self::Markdown => "Markdown", - Self::Workflow => "GitHub Actions workflow", - } - } - - const fn present_arguments(self) -> &'static [&'static str] { - match self { - Self::Markdown => &MARKDOWN_PRESENT, - Self::Workflow => &WORKFLOW_PRESENT, - } - } - - const fn deleted_arguments(self) -> &'static [&'static str] { - match self { - Self::Markdown => &MARKDOWN_DELETED, - Self::Workflow => &WORKFLOW_DELETED, - } - } - - const fn present_operation(self) -> &'static str { - match self { - Self::Markdown => "git Markdown present paths", - Self::Workflow => "git workflow present paths", - } - } - - const fn deleted_operation(self) -> &'static str { - match self { - Self::Markdown => "git Markdown deleted paths", - Self::Workflow => "git workflow deleted paths", + let paths = sources + .iter() + .map(|source| source.path().to_owned()) + .collect(); + Ok(Self { + kind, + paths, + sources, + }) } } } @@ -141,13 +91,13 @@ fn admit_paths<'a>( repository_root: &RepositoryRoot, paths: impl Iterator, kind: CorpusKind, -) -> Result, DocumentationError> { +) -> Result, DocumentationError> { let mut admitted = Vec::new(); let mut budget = CorpusByteBudget::default(); for path in paths { if let Some(source) = admit_path(repository_root, path, kind)? { - budget.admit(kind, &source)?; - admitted.push(source.path); + budget.admit(kind, source.path(), source.bytes())?; + admitted.push(source); } } Ok(admitted) @@ -169,17 +119,7 @@ fn admit_path( path: text.clone(), })?; match repository_root.open_file(&relative) { - Ok(file) => { - let bytes = file - .metadata() - .map_err(|source| DocumentationError::Inspect { - corpus: kind.label(), - path: text.clone(), - source, - })? - .len(); - Ok(Some(AdmittedSource { bytes, path: text })) - } + Ok(file) => Ok(Some(AdmittedSource::admit(&file, text, relative, kind)?)), Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { Ok(None) } @@ -195,6 +135,9 @@ fn admit_path( } } +#[cfg(test)] +#[path = "corpus/replacement_tests.rs"] +mod replacement_tests; #[cfg(test)] #[path = "corpus/tests.rs"] mod tests; diff --git a/xtask/src/documentation_integrity/corpus/byte_budget.rs b/xtask/src/documentation_integrity/corpus/byte_budget.rs index a3945be..405175c 100644 --- a/xtask/src/documentation_integrity/corpus/byte_budget.rs +++ b/xtask/src/documentation_integrity/corpus/byte_budget.rs @@ -1,6 +1,6 @@ //! This module owns documentation corpus byte-budget admission. -use super::{AdmittedSource, CorpusKind}; +use super::CorpusKind; use crate::documentation_integrity::error::DocumentationError; /// Maximum admitted bytes for one documentation source. @@ -19,19 +19,20 @@ impl CorpusByteBudget { pub(super) fn admit( &mut self, kind: CorpusKind, - source: &AdmittedSource, + path: &str, + bytes: u64, ) -> Result<(), DocumentationError> { - if source.bytes > CORPUS_FILE_MAX_BYTES { + if bytes > CORPUS_FILE_MAX_BYTES { return Err(DocumentationError::CorpusFileTooLarge { corpus: kind.label(), - path: source.path.clone(), + path: path.to_owned(), maximum: CORPUS_FILE_MAX_BYTES, - observed: source.bytes, + observed: bytes, }); } let observed = self .observed - .checked_add(source.bytes) + .checked_add(bytes) .ok_or_else(|| DocumentationError::CorpusSizeOverflow(kind.label()))?; if observed > CORPUS_MAX_BYTES { return Err(DocumentationError::CorpusTooLarge { diff --git a/xtask/src/documentation_integrity/corpus/replacement_tests.rs b/xtask/src/documentation_integrity/corpus/replacement_tests.rs new file mode 100644 index 0000000..aaa133c --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/replacement_tests.rs @@ -0,0 +1,36 @@ +//! This module owns documentation source-replacement regression evidence. + +use std::fs; + +use super::{SourceCorpus, test_repository::run_git}; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +#[test] +fn selected_source_replacement_refuses_the_admitted_corpus() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-source-replacement")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + fs::rename(root.join("selected.md"), root.join("retained.md"))?; + fs::write(root.join("selected.md"), "# Substitute\n")?; + + let result = corpus.verify_unchanged(&repository_root); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "selected.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/corpus/selection.rs b/xtask/src/documentation_integrity/corpus/selection.rs new file mode 100644 index 0000000..98d1e36 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/selection.rs @@ -0,0 +1,73 @@ +//! This module owns Git path-selection policy for documentation corpora. + +const MARKDOWN_PRESENT: [&str; 7] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + "*.md", +]; +const MARKDOWN_DELETED: [&str; 5] = ["ls-files", "-z", "--deleted", "--", "*.md"]; +const WORKFLOW_PRESENT: [&str; 8] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; +const WORKFLOW_DELETED: [&str; 6] = [ + "ls-files", + "-z", + "--deleted", + "--", + ".github/workflows/*.yml", + ".github/workflows/*.yaml", +]; + +#[derive(Clone, Copy)] +pub(super) enum CorpusKind { + Markdown, + Workflow, +} + +impl CorpusKind { + pub(super) const fn label(self) -> &'static str { + match self { + Self::Markdown => "Markdown", + Self::Workflow => "GitHub Actions workflow", + } + } + + pub(super) const fn present_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_PRESENT, + Self::Workflow => &WORKFLOW_PRESENT, + } + } + + pub(super) const fn deleted_arguments(self) -> &'static [&'static str] { + match self { + Self::Markdown => &MARKDOWN_DELETED, + Self::Workflow => &WORKFLOW_DELETED, + } + } + + pub(super) const fn present_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown present paths", + Self::Workflow => "git workflow present paths", + } + } + + pub(super) const fn deleted_operation(self) -> &'static str { + match self { + Self::Markdown => "git Markdown deleted paths", + Self::Workflow => "git workflow deleted paths", + } + } +} diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs new file mode 100644 index 0000000..0c76f22 --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -0,0 +1,112 @@ +//! This module owns retained identity evidence for one documentation source. + +use std::fs::{File, Metadata}; +use std::io; +use std::os::unix::fs::MetadataExt; +use std::path::PathBuf; + +use super::CorpusKind; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; + +pub(super) struct AdmittedSource { + identity: SourceIdentity, + path: String, + relative: PathBuf, +} + +#[derive(Eq, PartialEq)] +struct SourceIdentity { + device: u64, + inode: u64, + bytes: u64, + modified_seconds: i64, + modified_nanoseconds: i64, + changed_seconds: i64, + changed_nanoseconds: i64, +} + +impl AdmittedSource { + pub(super) fn admit( + file: &File, + path: String, + relative: PathBuf, + kind: CorpusKind, + ) -> Result { + let metadata = metadata(file, kind, &path)?; + Ok(Self { + identity: SourceIdentity::from(&metadata), + path, + relative, + }) + } + + pub(super) const fn bytes(&self) -> u64 { + self.identity.bytes + } + + pub(super) fn path(&self) -> &str { + &self.path + } + + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + kind: CorpusKind, + ) -> Result<(), DocumentationError> { + let current = match repository_root.open_file(&self.relative) { + Ok(file) => file, + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(changed(kind, &self.path)); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(DocumentationError::Inspect { + corpus: kind.label(), + path: self.path.clone(), + source, + }); + } + Err(OpenRepositoryFileError::NonRegular) => { + return Err(changed(kind, &self.path)); + } + }; + let current = SourceIdentity::from(&metadata(¤t, kind, &self.path)?); + if current == self.identity { + Ok(()) + } else { + Err(changed(kind, &self.path)) + } + } +} + +impl From<&Metadata> for SourceIdentity { + fn from(metadata: &Metadata) -> Self { + Self { + device: metadata.dev(), + inode: metadata.ino(), + bytes: metadata.len(), + modified_seconds: metadata.mtime(), + modified_nanoseconds: metadata.mtime_nsec(), + changed_seconds: metadata.ctime(), + changed_nanoseconds: metadata.ctime_nsec(), + } + } +} + +fn metadata(file: &File, kind: CorpusKind, path: &str) -> Result { + file.metadata() + .map_err(|source| DocumentationError::Inspect { + corpus: kind.label(), + path: path.to_owned(), + source, + }) +} + +fn changed(kind: CorpusKind, path: &str) -> DocumentationError { + DocumentationError::CorpusChanged { + corpus: kind.label(), + path: path.to_owned(), + } +} diff --git a/xtask/src/documentation_integrity/corpus/test_repository.rs b/xtask/src/documentation_integrity/corpus/test_repository.rs new file mode 100644 index 0000000..2fd37bc --- /dev/null +++ b/xtask/src/documentation_integrity/corpus/test_repository.rs @@ -0,0 +1,23 @@ +//! This module owns hermetic Git commands for corpus regression repositories. + +use std::error::Error; +use std::path::Path; +use std::process::Command; + +pub(in crate::documentation_integrity) fn run_git( + root: &Path, + arguments: &[&str], +) -> Result<(), Box> { + let output = Command::new("git") + .args(arguments) + .current_dir(root) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .output()?; + if output.status.success() { + Ok(()) + } else { + Err(format!("git fixture command failed: {arguments:?}").into()) + } +} diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs index 68a8221..893c12a 100644 --- a/xtask/src/documentation_integrity/corpus/tests.rs +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::Path; use std::process::Command; -use super::{CorpusKind, SourceCorpus, admit_path}; +use super::{CorpusKind, SourceCorpus, admit_path, test_repository::run_git}; use crate::documentation_integrity::error::DocumentationError; use crate::git_inventory::GitPath; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; @@ -16,7 +16,7 @@ fn markdown_corpus_is_the_sorted_present_repository_set() -> Result<(), Box Result<(), Box Result<(), Box> { let directory = TestDirectory::create("documentation-global-ignore")?; let root = directory.path(); - run_git(root, &["init", "--quiet"])?; + run_git(root, &["init", "--quiet", "--template="])?; write(root, "tracked.md", "# Tracked\n")?; write(root, "new.md", "# New\n")?; write(root, "global-ignore", "*.md\n")?; @@ -100,7 +100,7 @@ fn transient_repository_replacement_cannot_redirect_the_corpus() let root = directory.path().join("repository"); let retained = directory.path().join("retained"); fs::create_dir(&root)?; - run_git(&root, &["init", "--quiet"])?; + run_git(&root, &["init", "--quiet", "--template="])?; write(&root, "original.md", "# Original\n")?; let repository_root = RepositoryRoot::open(&root)?; let process_directory = repository_root.process_directory()?; @@ -108,7 +108,7 @@ fn transient_repository_replacement_cannot_redirect_the_corpus() fs::rename(&root, &retained)?; fs::create_dir(&root)?; - run_git(&root, &["init", "--quiet"])?; + run_git(&root, &["init", "--quiet", "--template="])?; write(&root, "substitute.md", "# Substitute\n")?; let corpus = SourceCorpus::markdown(&repository_root, &process_directory); @@ -130,7 +130,7 @@ fn symlinked_markdown_is_refused() -> Result<(), Box> { let directory = TestDirectory::create("documentation-markdown-symlink")?; let root = directory.path(); - run_git(root, &["init", "--quiet"])?; + run_git(root, &["init", "--quiet", "--template="])?; write(root, "target.txt", "target\n")?; symlink("target.txt", root.join("linked.md"))?; @@ -186,19 +186,23 @@ fn oversized_markdown_is_refused_before_external_validation() fn aggregate_documentation_bytes_are_bounded_with_checked_accounting() -> Result<(), Box> { let mut budget = super::CorpusByteBudget::default(); - let source = super::AdmittedSource { - bytes: super::CORPUS_FILE_MAX_BYTES, - path: String::from("bounded.md"), - }; for _ in 0..16 { - budget.admit(CorpusKind::Markdown, &source)?; + budget.admit( + CorpusKind::Markdown, + "bounded.md", + super::CORPUS_FILE_MAX_BYTES, + )?; } let observed = super::CORPUS_MAX_BYTES .checked_add(super::CORPUS_FILE_MAX_BYTES) .ok_or("aggregate corpus bound overflow")?; assert!(matches!( - budget.admit(CorpusKind::Markdown, &source), + budget.admit( + CorpusKind::Markdown, + "bounded.md", + super::CORPUS_FILE_MAX_BYTES, + ), Err(DocumentationError::CorpusTooLarge { corpus: "Markdown", maximum: super::CORPUS_MAX_BYTES, @@ -213,7 +217,7 @@ fn aggregate_documentation_bytes_are_bounded_with_checked_accounting() fn fifo_workflow_is_refused() -> Result<(), Box> { let directory = TestDirectory::create("documentation-workflow-fifo")?; let root = directory.path(); - run_git(root, &["init", "--quiet"])?; + run_git(root, &["init", "--quiet", "--template="])?; write(root, ".github/workflows/blocking.yml", "name: Blocking\n")?; run_git(root, &["add", ".github/workflows/blocking.yml"])?; let fifo = root.join(".github/workflows/blocking.yml"); @@ -258,18 +262,6 @@ fn non_utf8_markdown_path_is_refused() -> Result<(), Box> Ok(()) } -fn run_git(root: &Path, arguments: &[&str]) -> Result<(), Box> { - let output = Command::new("git") - .args(arguments) - .current_dir(root) - .output()?; - if output.status.success() { - Ok(()) - } else { - Err(format!("git fixture command failed: {arguments:?}").into()) - } -} - fn write(root: &Path, relative: &str, contents: &str) -> Result<(), Box> { let path = root.join(relative); if let Some(parent) = path.parent() { diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index ff5773d..9a484de 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -28,6 +28,10 @@ pub(crate) enum DocumentationError { maximum: u64, observed: u64, }, + CorpusChanged { + corpus: &'static str, + path: String, + }, EmptyCorpus(&'static str), GitInventory(GitInventoryError), Inspect { @@ -138,6 +142,7 @@ impl Error for DocumentationError { Self::CorpusFileTooLarge { .. } | Self::CorpusSizeOverflow(_) | Self::CorpusTooLarge { .. } + | Self::CorpusChanged { .. } | Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index b0caf2c..826828b 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -8,6 +8,7 @@ use super::DocumentationError; #[derive(Clone, Copy)] enum SourcePathDiagnostic { + Changed, Inspect, Invalid, NonRegular, @@ -25,27 +26,11 @@ impl fmt::Display for DocumentationError { Self::CheckFailures { first, second } => { write!(formatter, "{first}; additionally: {second}") } - Self::CorpusFileTooLarge { - corpus, - path, - maximum, - observed, - } => write!( - formatter, - "{corpus} source `{path}` is {observed} bytes; maximum is {maximum}" - ), - Self::CorpusSizeOverflow(corpus) => { - write!(formatter, "{corpus} corpus byte count overflowed") - } - Self::CorpusTooLarge { - corpus, - maximum, - observed, - } => write!( - formatter, - "{corpus} corpus is {observed} bytes; maximum is {maximum}" - ), - Self::EmptyCorpus(label) => write!(formatter, "the {label} corpus is empty"), + error @ (Self::CorpusFileTooLarge { .. } + | Self::CorpusSizeOverflow(_) + | Self::CorpusTooLarge { .. } + | Self::CorpusChanged { .. } + | Self::EmptyCorpus(_)) => corpus(formatter, error), Self::GitInventory(error) => write!(formatter, "{error}"), Self::Inspect { corpus, path, .. } => { source_path(formatter, SourcePathDiagnostic::Inspect, corpus, path) @@ -83,6 +68,38 @@ impl fmt::Display for DocumentationError { } } +fn corpus(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { + match error { + DocumentationError::CorpusFileTooLarge { + corpus, + path, + maximum, + observed, + } => write!( + formatter, + "{corpus} source `{path}` is {observed} bytes; maximum is {maximum}" + ), + DocumentationError::CorpusSizeOverflow(corpus) => { + write!(formatter, "{corpus} corpus byte count overflowed") + } + DocumentationError::CorpusTooLarge { + corpus, + maximum, + observed, + } => write!( + formatter, + "{corpus} corpus is {observed} bytes; maximum is {maximum}" + ), + DocumentationError::CorpusChanged { corpus, path } => { + source_path(formatter, SourcePathDiagnostic::Changed, corpus, path) + } + DocumentationError::EmptyCorpus(label) => { + write!(formatter, "the {label} corpus is empty") + } + _ => Err(fmt::Error), + } +} + fn repository_file(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { match error { DocumentationError::RepositoryFileEncoding { path, .. } => { @@ -165,6 +182,9 @@ fn source_path( path: &str, ) -> fmt::Result { match diagnostic { + SourcePathDiagnostic::Changed => { + write!(formatter, "{corpus} source changed during validation: `")?; + } SourcePathDiagnostic::Inspect => write!(formatter, "cannot inspect {corpus} source `")?, SourcePathDiagnostic::Invalid => { write!(formatter, "{corpus} corpus contains an unsafe path `")?; diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index a123eb9..f4b5e15 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -1,13 +1,17 @@ //! This module owns bounded execution of admitted documentation tools. +mod corpus_guard; + use std::process::{Command, Stdio}; use std::time::Duration; use crate::bounded_process::{self, ProcessOutput}; -use crate::repository_file::RepositoryProcessDirectory; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; +use super::corpus::SourceCorpus; use super::error::DocumentationError; use super::tool::DocumentationTool; +use corpus_guard::CorpusGuardedRunner; const TOOL_DEADLINE: Duration = Duration::from_mins(2); @@ -25,14 +29,14 @@ struct ExternalToolRunner<'a> { pub(super) fn run( process_directory: &RepositoryProcessDirectory, - markdown: &[String], - workflows: &[String], + repository_root: &RepositoryRoot, + markdown: &SourceCorpus, + workflows: &SourceCorpus, ) -> Result<(), DocumentationError> { - run_with( - &mut ExternalToolRunner { process_directory }, - markdown, - workflows, - ) + let corpora = [markdown, workflows]; + let external = ExternalToolRunner { process_directory }; + let mut runner = CorpusGuardedRunner::new(external, repository_root, &corpora); + run_with(&mut runner, markdown.paths(), workflows.paths()) } fn run_with( diff --git a/xtask/src/documentation_integrity/execution/corpus_guard.rs b/xtask/src/documentation_integrity/execution/corpus_guard.rs new file mode 100644 index 0000000..bec350c --- /dev/null +++ b/xtask/src/documentation_integrity/execution/corpus_guard.rs @@ -0,0 +1,51 @@ +//! This module owns corpus revalidation around external tool execution. + +use crate::bounded_process::ProcessOutput; +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::RepositoryRoot; + +use super::{DocumentationTool, ToolRunner}; + +pub(super) struct CorpusGuardedRunner<'a, Runner> { + corpora: &'a [&'a SourceCorpus], + inner: Runner, + repository_root: &'a RepositoryRoot, +} + +impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { + pub(super) const fn new( + inner: Runner, + repository_root: &'a RepositoryRoot, + corpora: &'a [&'a SourceCorpus], + ) -> Self { + Self { + corpora, + inner, + repository_root, + } + } + + fn verify(&self) -> Result<(), DocumentationError> { + for corpus in self.corpora { + corpus.verify_unchanged(self.repository_root)?; + } + Ok(()) + } +} + +impl ToolRunner for CorpusGuardedRunner<'_, Runner> +where + Runner: ToolRunner, +{ + fn capture( + &mut self, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + self.verify()?; + let result = self.inner.capture(tool, arguments); + self.verify()?; + result + } +} diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index a55b55c..2805a45 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -1,7 +1,13 @@ use std::collections::VecDeque; +use std::fs; +use std::path::PathBuf; use crate::bounded_process::ProcessOutput; +use crate::documentation_integrity::corpus::{SourceCorpus, test_repository::run_git}; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; +use super::corpus_guard::CorpusGuardedRunner; use super::{DocumentationError, DocumentationTool, ToolRunner}; struct RecordingRunner { @@ -9,6 +15,11 @@ struct RecordingRunner { outputs: VecDeque, } +struct ReplacingRunner { + selected: PathBuf, + retained: PathBuf, +} + #[test] fn admitted_tools_run_with_exact_arguments_and_silent_success() { let mut runner = RecordingRunner::new([ @@ -117,6 +128,38 @@ fn unreviewed_version_stops_before_tool_execution() { assert_eq!(runner.calls.len(), 1); } +#[test] +fn corpus_guard_refuses_a_source_restored_after_transient_replacement() +-> Result<(), Box> { + let directory = TestDirectory::create("documentation-transient-source")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + let corpora = [&corpus]; + let replacing = ReplacingRunner { + selected: root.join("selected.md"), + retained: root.join("retained.md"), + }; + let mut runner = CorpusGuardedRunner::new(replacing, &repository_root, &corpora); + + let result = runner.capture(DocumentationTool::Markdownlint, &[]); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "selected.md" + )); + drop(runner); + drop(corpus); + directory.close()?; + Ok(()) +} + impl RecordingRunner { fn new(outputs: impl IntoIterator) -> Self { Self { @@ -142,6 +185,32 @@ impl ToolRunner for RecordingRunner { } } +impl ToolRunner for ReplacingRunner { + fn capture( + &mut self, + _tool: DocumentationTool, + _arguments: &[String], + ) -> Result { + fs::rename(&self.selected, &self.retained) + .map_err(|source| fixture_io("retain selected source", source))?; + fs::write(&self.selected, "# Substitute\n") + .map_err(|source| fixture_io("write substitute source", source))?; + fs::remove_file(&self.selected) + .map_err(|source| fixture_io("remove substitute source", source))?; + fs::rename(&self.retained, &self.selected) + .map_err(|source| fixture_io("restore selected source", source))?; + Ok(success()) + } +} + +fn fixture_io(requirement: &'static str, source: std::io::Error) -> DocumentationError { + DocumentationError::Inspect { + corpus: "test", + path: requirement.to_owned(), + source, + } +} + fn version(tool: DocumentationTool) -> ProcessOutput { ProcessOutput { code: Some(0), From ad7349c50a78544ff21030235cfb614caf788b05 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:18:47 -0700 Subject: [PATCH 074/113] Fix: track the current workflow parser --- xtask/tests/documentation_cli_contract.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xtask/tests/documentation_cli_contract.rs b/xtask/tests/documentation_cli_contract.rs index 74251c1..fe8d20e 100644 --- a/xtask/tests/documentation_cli_contract.rs +++ b/xtask/tests/documentation_cli_contract.rs @@ -27,18 +27,18 @@ fn documentation_error_formatter_stays_below_the_hard_function_limit() -> Result } #[test] -fn workflow_parser_stays_below_the_hard_function_limit() -> Result<(), &'static str> { +fn workflow_admission_parser_stays_below_the_hard_function_limit() -> Result<(), &'static str> { let (_, after_signature) = WORKFLOW_CONTRACT .split_once( - "fn documentation_runs(workflow: &str) -> Result, DocumentationError> {", + "fn admitted_steps(workflow: &str) -> Result, DocumentationError> {", ) .ok_or("workflow contract must retain its parser")?; let (body, _) = after_signature - .split_once("\n}\n\nfn documentation_steps") + .split_once("\n}\n\nfn triggers_are_reviewed") .ok_or("workflow parser must remain a directly inspectable function")?; assert!( body.lines().count() <= 59, - "documentation_runs exceeds the 60-line hard limit" + "admitted_steps exceeds the 60-line hard limit" ); Ok(()) } From 9d621c1740432d3b3fc2c675f20de6ce7cdccc0b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:26:05 -0700 Subject: [PATCH 075/113] Fix: bound Git inventory processes --- CHANGELOG.md | 4 +- ...007-terminal-signal-process-group-guard.md | 8 + docs/dependencies/documentation-toolchain.md | 6 + xtask/src/bounded_process.rs | 10 +- xtask/src/bounded_process/capture.rs | 40 ++- xtask/src/bounded_process/capture_limit.rs | 29 +++ xtask/src/git_inventory/error.rs | 36 +-- xtask/src/git_inventory/path_stream.rs | 5 +- xtask/src/git_inventory/process.rs | 232 ++++-------------- xtask/src/git_inventory/process/tests.rs | 92 ++++--- xtask/tests/source_policy_contract.rs | 30 ++- 11 files changed, 235 insertions(+), 257 deletions(-) create mode 100644 xtask/src/bounded_process/capture_limit.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 34cb432..53fbdec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,8 +34,8 @@ after its public API and format compatibility policies are established. to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time drift before and after each external tool, - and applies one deadline across captured and inherited child execution and - output collection. + and applies a two-minute deadline across Git inventory, validation-tool + execution, and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation Git inventory and tools start from one retained repository directory handle, diff --git a/docs/adr/0007-terminal-signal-process-group-guard.md b/docs/adr/0007-terminal-signal-process-group-guard.md index 9d52445..4ba8cfb 100644 --- a/docs/adr/0007-terminal-signal-process-group-guard.md +++ b/docs/adr/0007-terminal-signal-process-group-guard.md @@ -18,6 +18,9 @@ Captured output creates a second wait boundary. A child leader can exit while a descendant retains an inherited pipe, so guarding only the child wait does not cover the complete operation. +Git path inventory has the same boundary. A damaged repository can stall Git, +and a descendant can retain either output pipe after Git exits. + ## Decision While an external repository task is active, `xtask` installs one process-wide @@ -26,6 +29,11 @@ thread records the first signal for every active operation. Child waits and captured-output readers poll that state at the existing ten-millisecond process interval. +Git path inventory and documentation validators use the same two-minute +deadline, process-group isolation, signal guard, and reader cleanup. Git keeps +its 16 MiB path-stream and 64 KiB diagnostic bounds through explicit +per-stream capture limits. + An observed terminal signal becomes a typed `ProcessError::Interrupted` refusal. The normal failure path then sends `SIGKILL` to the dedicated child process group, kills and reaps the child, and joins captured-output readers diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index b84db5e..04b74aa 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -51,6 +51,12 @@ deletions; Git-trackable nonregular paths such as symlinks and tracked paths replaced by FIFOs are refused. Non-trackable special files cannot enter the Git-selected corpus. +Each Git path inventory runs in a dedicated process group under a two-minute +deadline that covers the child and both output readers. Standard output retains +at most the 16 MiB path-stream bound, and diagnostics retain at most 64 KiB. A +timeout, terminal signal, reader failure, or exceeded bound terminates the +whole group and reaps the child before the task refuses. + Git inventory and each validation tool start through one retained repository directory handle. Child-only setup changes directory through that handle after fork and before exec; the parent working directory does not change. Replacing diff --git a/xtask/src/bounded_process.rs b/xtask/src/bounded_process.rs index 2dfb7bd..25e8ea7 100644 --- a/xtask/src/bounded_process.rs +++ b/xtask/src/bounded_process.rs @@ -1,6 +1,7 @@ //! This module owns bounded external child-process collection. mod capture; +mod capture_limit; mod cleanup; mod deadline; mod error; @@ -13,7 +14,8 @@ use std::process::Command; use std::time::Duration; use capture::wait_for_child; -pub(crate) use capture::{capture, capture_with}; +pub(crate) use capture::{capture, capture_with, capture_with_limits}; +pub(crate) use capture_limit::CaptureLimits; use deadline::ProcessDeadline; pub(crate) use error::ProcessError; use interrupt::InterruptGuard; @@ -21,9 +23,9 @@ use reader::ReaderWorker; /// The completed child status and any output retained by the selected mode. /// -/// Captured execution retains at most one mebibyte per output stream. -/// Inherited execution leaves both byte vectors empty because the child writes -/// directly to the parent's configured streams. +/// Captured execution retains at most the selected limit for each output +/// stream. Inherited execution leaves both byte vectors empty because the child +/// writes directly to the parent's configured streams. pub(crate) struct ProcessOutput { /// The platform exit code, or `None` when the process ended by signal. pub(crate) code: Option, diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 0a47a4b..07d355c 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -6,10 +6,12 @@ use std::thread; use std::time::Duration; use super::cleanup::{cleanup_process, join_after_cleanup, join_readers}; -use super::{InterruptGuard, ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker}; +use super::{ + CaptureLimits, InterruptGuard, ProcessDeadline, ProcessError, ProcessOutput, ReaderWorker, +}; use crate::process_output::BoundedBytes; -const OUTPUT_LIMIT: usize = 1_048_576; +const DEFAULT_CAPTURE_LIMITS: CaptureLimits = CaptureLimits::new(1_048_576, 1_048_576); const PROCESS_POLL_INTERVAL: Duration = Duration::from_millis(10); /// Runs a child synchronously and captures bounded standard output and error. @@ -26,20 +28,39 @@ pub(crate) fn capture( capture_with(program, command, deadline, Command::spawn) } +/// Runs one captured child through an injected spawn boundary. +/// +/// This variant retains the default one-mebibyte limit for each stream while +/// allowing a capability-bound caller to own the actual spawn operation. pub(crate) fn capture_with( program: &'static str, command: &mut Command, deadline: Option, spawn: impl FnOnce(&mut Command) -> Result, +) -> Result { + capture_with_limits(program, command, deadline, DEFAULT_CAPTURE_LIMITS, spawn) +} + +/// Runs one captured child with exact independent stream limits. +/// +/// The deadline covers child execution and both reader workers. Every failure +/// terminates the dedicated process group and joins the workers before return. +pub(crate) fn capture_with_limits( + program: &'static str, + command: &mut Command, + deadline: Option, + limits: CaptureLimits, + spawn: impl FnOnce(&mut Command) -> Result, ) -> Result { let deadline = ProcessDeadline::new(program, deadline)?; let interrupts = InterruptGuard::begin(program)?; - CapturedProcess::start(program, command, spawn, interrupts)?.finish(program, &deadline) + CapturedProcess::start(program, command, spawn, interrupts, limits)?.finish(program, &deadline) } struct CapturedProcess { child: Child, interrupts: InterruptGuard, + limits: CaptureLimits, stderr: ReaderWorker, stdout: ReaderWorker, } @@ -50,6 +71,7 @@ impl CapturedProcess { command: &mut Command, spawn: impl FnOnce(&mut Command) -> Result, interrupts: InterruptGuard, + limits: CaptureLimits, ) -> Result { command .stdout(Stdio::piped()) @@ -74,11 +96,11 @@ impl CapturedProcess { return Err(cleanup_process(&mut child, error)); } }; - let stdout = match ReaderWorker::start(program, "stdout", stdout, OUTPUT_LIMIT) { + let stdout = match ReaderWorker::start(program, "stdout", stdout, limits.stdout_bytes()) { Ok(reader) => reader, Err(error) => return Err(cleanup_process(&mut child, error)), }; - let stderr = match ReaderWorker::start(program, "stderr", stderr, OUTPUT_LIMIT) { + let stderr = match ReaderWorker::start(program, "stderr", stderr, limits.stderr_bytes()) { Ok(reader) => reader, Err(error) => { let error = cleanup_process(&mut child, error); @@ -88,6 +110,7 @@ impl CapturedProcess { Ok(Self { child, interrupts, + limits, stderr, stdout, }) @@ -117,8 +140,8 @@ impl CapturedProcess { if let Err(error) = self.stderr.join() { return Err(cleanup_process(&mut self.child, error)); } - refuse_exceeded(program, "stdout", &stdout) - .and_then(|()| refuse_exceeded(program, "stderr", &stderr)) + refuse_exceeded(program, "stdout", self.limits.stdout_bytes(), &stdout) + .and_then(|()| refuse_exceeded(program, "stderr", self.limits.stderr_bytes(), &stderr)) .map_err(|error| cleanup_process(&mut self.child, error))?; if let Some(error) = self.interrupts.refusal(program) { return Err(cleanup_process(&mut self.child, error)); @@ -175,13 +198,14 @@ pub(super) fn wait_for_child( const fn refuse_exceeded( program: &'static str, stream: &'static str, + maximum: usize, output: &BoundedBytes, ) -> Result<(), ProcessError> { if output.exceeded { Err(ProcessError::OutputLimit { program, stream, - maximum: OUTPUT_LIMIT, + maximum, }) } else { Ok(()) diff --git a/xtask/src/bounded_process/capture_limit.rs b/xtask/src/bounded_process/capture_limit.rs new file mode 100644 index 0000000..d6040b2 --- /dev/null +++ b/xtask/src/bounded_process/capture_limit.rs @@ -0,0 +1,29 @@ +//! This module owns retained-byte limits for captured child streams. + +/// Maximum retained bytes for one child standard-output and standard-error pair. +#[derive(Clone, Copy)] +pub(crate) struct CaptureLimits { + stderr_bytes: usize, + stdout_bytes: usize, +} + +impl CaptureLimits { + /// Creates exact independent limits for standard output and standard error. + #[must_use] + pub(crate) const fn new(stdout_bytes: usize, stderr_bytes: usize) -> Self { + Self { + stderr_bytes, + stdout_bytes, + } + } + + /// Returns the standard-error retained-byte limit. + pub(super) const fn stderr_bytes(self) -> usize { + self.stderr_bytes + } + + /// Returns the standard-output retained-byte limit. + pub(super) const fn stdout_bytes(self) -> usize { + self.stdout_bytes + } +} diff --git a/xtask/src/git_inventory/error.rs b/xtask/src/git_inventory/error.rs index 6e93bbe..f15ce42 100644 --- a/xtask/src/git_inventory/error.rs +++ b/xtask/src/git_inventory/error.rs @@ -5,6 +5,7 @@ use std::fmt::{self, Write as _}; use std::io; use std::string::FromUtf8Error; +use crate::bounded_process::ProcessError; use crate::diagnostic::escaped_controls; #[derive(Clone, Copy)] @@ -18,11 +19,6 @@ pub(crate) enum GitOutputUnit { /// A typed failure while listing or decoding repository paths from Git. pub(crate) enum GitInventoryError { - /// Cleanup failed after an earlier inventory failure was already detected. - Cleanup { - primary: Box, - cleanup: Box, - }, /// Git emitted the same path record more than once. DuplicatePath(Vec), /// Git emitted an empty NUL-framed path record. @@ -48,10 +44,10 @@ pub(crate) enum GitInventoryError { }, /// Git ended its output with bytes not terminated by a NUL delimiter. OutputFraming { operation: &'static str }, - /// A Git child configured for capture did not expose a requested pipe. - Pipe { + /// Deadline-bounded Git process execution failed. + Process { operation: &'static str, - stream: &'static str, + source: ProcessError, }, /// A named operating-system action for the Git child failed. Run { @@ -59,8 +55,6 @@ pub(crate) enum GitInventoryError { action: &'static str, source: io::Error, }, - /// The concurrent diagnostic-reader thread stopped by panicking. - Worker { operation: &'static str }, } impl fmt::Debug for GitInventoryError { @@ -72,12 +66,6 @@ impl fmt::Debug for GitInventoryError { impl fmt::Display for GitInventoryError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Cleanup { primary, cleanup } => { - write!( - formatter, - "{primary}; additionally, cleanup failed: {cleanup}" - ) - } Self::DuplicatePath(path) => { formatter.write_str("git returned duplicate path `")?; escaped_bytes(formatter, path)?; @@ -113,18 +101,12 @@ impl fmt::Display for GitInventoryError { "`{operation}` returned a non-NUL-terminated path" ) } - Self::Pipe { operation, stream } => { - write!(formatter, "`{operation}` did not provide its {stream} pipe") + Self::Process { operation, source } => { + write!(formatter, "{source} while running `{operation}`") } Self::Run { operation, action, .. } => write!(formatter, "cannot {action} `{operation}`"), - Self::Worker { operation } => { - write!( - formatter, - "`{operation}` diagnostic reader stopped unexpectedly" - ) - } } } } @@ -141,16 +123,14 @@ impl GitOutputUnit { impl Error for GitInventoryError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Cleanup { primary, .. } => Some(primary), Self::DiagnosticEncoding { source, .. } => Some(source), + Self::Process { source, .. } => Some(source), Self::Run { source, .. } => Some(source), Self::DuplicatePath(_) | Self::EmptyPath { .. } | Self::Failed { .. } | Self::OutputBound { .. } - | Self::OutputFraming { .. } - | Self::Pipe { .. } - | Self::Worker { .. } => None, + | Self::OutputFraming { .. } => None, } } } diff --git a/xtask/src/git_inventory/path_stream.rs b/xtask/src/git_inventory/path_stream.rs index a6f5311..cb4597b 100644 --- a/xtask/src/git_inventory/path_stream.rs +++ b/xtask/src/git_inventory/path_stream.rs @@ -5,9 +5,12 @@ use std::io::Read; use super::{GitInventoryError, GitOutputUnit}; +/// Maximum retained bytes in one complete Git path stream. +pub(super) const GIT_PATH_STREAM_LIMIT_BYTES: usize = 16_777_216; + const GIT_PATH_LIMITS: GitPathLimits = GitPathLimits { path_bytes: 4_096, - stream_bytes: 16_777_216, + stream_bytes: GIT_PATH_STREAM_LIMIT_BYTES, paths: 100_000, }; diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index 25925b3..c9687b8 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -1,30 +1,25 @@ -//! This module owns bounded Git path process execution and collection. +//! This module owns deadline-bounded Git path process execution. use std::collections::BTreeSet; use std::io; use std::path::Path; -use std::process::{Child, ChildStdout, Command, Stdio}; -use std::thread::{self, JoinHandle}; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; -use crate::process_output::{BoundedBytes, bounded_bytes}; +use crate::bounded_process::{self, CaptureLimits, ProcessError}; -use super::path_stream::{GitPath, read_paths}; +use super::path_stream::{GIT_PATH_STREAM_LIMIT_BYTES, GitPath, read_paths}; use super::{GitInventoryError, GitOutputUnit}; +const GIT_DEADLINE: Duration = Duration::from_mins(2); const GIT_DIAGNOSTIC_LIMIT_BYTES: usize = 65_536; +const GIT_CAPTURE_LIMITS: CaptureLimits = + CaptureLimits::new(GIT_PATH_STREAM_LIMIT_BYTES, GIT_DIAGNOSTIC_LIMIT_BYTES); -struct GitProcess { - child: Child, - diagnostic_worker: JoinHandle>, - stdout: ChildStdout, -} - -/// Lists repository paths without allowing either child pipe to block. +/// Lists repository paths through a deadline-bounded Git process group. /// -/// Standard error is drained concurrently before standard output is read. -/// Collection reads standard output before requesting termination, waits for -/// the child before joining the diagnostic reader, and then preserves the -/// established error precedence. +/// The adapter materializes at most the 16 MiB path-stream bound before +/// deterministic NUL-framed decoding. pub(crate) fn paths( repository_root: &Path, arguments: &[&str], @@ -35,189 +30,68 @@ pub(crate) fn paths( }) } +/// Lists paths through an injected capability-bound spawn operation. +/// +/// The adapter materializes at most the 16 MiB path-stream bound before +/// deterministic NUL-framed decoding. pub(crate) fn paths_with( arguments: &[&str], operation: &'static str, spawn: impl FnOnce(&mut Command) -> Result, ) -> Result, GitInventoryError> { - let process = start_git(arguments, operation, spawn)?; - let paths = read_paths(process.stdout, operation); - collect_git_result(process.child, process.diagnostic_worker, paths, operation) + paths_with_deadline(arguments, operation, GIT_DEADLINE, spawn) } -fn start_git( +fn paths_with_deadline( arguments: &[&str], operation: &'static str, + deadline: Duration, spawn: impl FnOnce(&mut Command) -> Result, -) -> Result { - let mut command = Command::new("git"); - command - .args(arguments) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let mut child = spawn(&mut command).map_err(|source| GitInventoryError::Run { - operation, - action: "start", - source, - })?; - let Some(stdout) = child.stdout.take() else { - let primary = GitInventoryError::Pipe { - operation, - stream: "stdout", - }; - return Err(cleanup_child(&mut child, operation, primary)); - }; - let Some(stderr) = child.stderr.take() else { - let primary = GitInventoryError::Pipe { - operation, - stream: "stderr", - }; - return Err(cleanup_child(&mut child, operation, primary)); - }; - let diagnostic_worker = thread::Builder::new() - .name(String::from("xtask-git-diagnostic")) - .spawn(move || bounded_bytes(stderr, GIT_DIAGNOSTIC_LIMIT_BYTES)); - let diagnostic_worker = match diagnostic_worker { - Ok(worker) => worker, - Err(source) => { - let primary = GitInventoryError::Run { - operation, - action: "start the diagnostic reader for", - source, - }; - return Err(cleanup_child(&mut child, operation, primary)); - } - }; - Ok(GitProcess { - child, - diagnostic_worker, - stdout, - }) -} - -fn collect_git_result( - mut child: Child, - diagnostic_worker: JoinHandle>, - paths: Result, GitInventoryError>, - operation: &'static str, ) -> Result, GitInventoryError> { - let stop = if paths.is_err() { - request_stop(&mut child, operation) + let mut command = Command::new("git"); + command.args(arguments).stdin(Stdio::null()); + let output = bounded_process::capture_with_limits( + "git", + &mut command, + Some(deadline), + GIT_CAPTURE_LIMITS, + spawn, + ) + .map_err(|source| process_failure(operation, source))?; + let paths = read_paths(output.stdout.as_slice(), operation)?; + if output.succeeded { + Ok(paths) } else { - Ok(()) - }; - let status = child.wait().map_err(|source| GitInventoryError::Run { - operation, - action: "wait for", - source, - }); - let diagnostic = diagnostic_worker.join(); - let paths = match paths { - Ok(paths) => paths, - Err(primary) => { - return Err(preserve_collection_failure( - primary, stop, status, diagnostic, operation, - )); - } - }; - let status = match status { - Ok(status) => status, - Err(primary) => { - return Err(preserve_diagnostic_failure(primary, diagnostic, operation)); - } - }; - let diagnostic = diagnostic - .map_err(|_| GitInventoryError::Worker { operation })? - .map_err(|source| GitInventoryError::Run { - operation, - action: "read diagnostics from", - source, - })?; - if diagnostic.exceeded { - return Err(GitInventoryError::OutputBound { - operation, - stream: "diagnostic bytes", - maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, - unit: GitOutputUnit::Bytes, - }); + Err(git_failure(operation, output.code, output.stderr)) } - if !status.success() { - return Err(git_failure(operation, status.code(), diagnostic.bytes)); - } - Ok(paths) } -fn request_stop(child: &mut Child, operation: &'static str) -> Result<(), GitInventoryError> { - child - .kill() - .or_else(|source| { - if source.kind() == io::ErrorKind::InvalidInput { - Ok(()) - } else { - Err(source) - } - }) - .map_err(|source| GitInventoryError::Run { - operation, - action: "stop", - source, - }) +fn process_failure(operation: &'static str, source: ProcessError) -> GitInventoryError { + match source { + ProcessError::OutputLimit { + stream: "stdout", + maximum, + .. + } => output_bound(operation, "path stream bytes", maximum), + ProcessError::OutputLimit { + stream: "stderr", + maximum, + .. + } => output_bound(operation, "diagnostic bytes", maximum), + source => GitInventoryError::Process { operation, source }, + } } -fn cleanup_child( - child: &mut Child, +const fn output_bound( operation: &'static str, - primary: GitInventoryError, + stream: &'static str, + maximum: usize, ) -> GitInventoryError { - let stop = request_stop(child, operation); - let wait = child.wait().map_err(|source| GitInventoryError::Run { + GitInventoryError::OutputBound { operation, - action: "wait for", - source, - }); - let primary = preserve_error(primary, stop); - preserve_error(primary, wait.map(|_| ())) -} - -fn preserve_collection_failure( - primary: GitInventoryError, - stop: Result<(), GitInventoryError>, - status: Result, - diagnostic: thread::Result>, - operation: &'static str, -) -> GitInventoryError { - let primary = preserve_error(primary, stop); - let primary = preserve_error(primary, status.map(|_| ())); - preserve_diagnostic_failure(primary, diagnostic, operation) -} - -fn preserve_diagnostic_failure( - primary: GitInventoryError, - diagnostic: thread::Result>, - operation: &'static str, -) -> GitInventoryError { - let cleanup = match diagnostic { - Ok(Ok(_)) => return primary, - Ok(Err(source)) => GitInventoryError::Run { - operation, - action: "read diagnostics from", - source, - }, - Err(_) => GitInventoryError::Worker { operation }, - }; - preserve_error(primary, Err(cleanup)) -} - -fn preserve_error( - primary: GitInventoryError, - cleanup: Result<(), GitInventoryError>, -) -> GitInventoryError { - match cleanup { - Ok(()) => primary, - Err(cleanup) => GitInventoryError::Cleanup { - primary: Box::new(primary), - cleanup: Box::new(cleanup), - }, + stream, + maximum, + unit: GitOutputUnit::Bytes, } } diff --git a/xtask/src/git_inventory/process/tests.rs b/xtask/src/git_inventory/process/tests.rs index 0b114cc..680129c 100644 --- a/xtask/src/git_inventory/process/tests.rs +++ b/xtask/src/git_inventory/process/tests.rs @@ -1,9 +1,17 @@ -//! This module owns adversarial Git diagnostic-bound tests. +//! This module owns Git process deadline and diagnostic regression evidence. -use std::io::{self, Cursor}; +use std::env; +use std::process::{Command, Stdio}; +use std::time::Duration; -use super::{GitInventoryError, git_failure, preserve_error}; -use crate::process_output::bounded_bytes; +use std::os::unix::process::CommandExt; + +use super::{GIT_DIAGNOSTIC_LIMIT_BYTES, git_failure, paths_with_deadline, process_failure}; +use crate::bounded_process::ProcessError; +use crate::git_inventory::GitInventoryError; + +const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_GIT_CHILD"; +const PARKED_CHILD_TEST: &str = "git_inventory::process::tests::process_child_parks_indefinitely"; #[test] fn git_diagnostic_encoding_failure_retains_exit_status() { @@ -19,38 +27,62 @@ fn git_diagnostic_encoding_failure_retains_exit_status() { } #[test] -fn git_diagnostics_are_drained_but_only_the_bound_is_retained() { - let result = bounded_bytes(Cursor::new(b"abcdef"), 3); +fn git_diagnostic_limit_maps_to_the_inventory_boundary() { + let error = process_failure( + "test diagnostics", + ProcessError::OutputLimit { + program: "git", + stream: "stderr", + maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, + }, + ); assert!(matches!( - result, - Ok(ref diagnostic) if diagnostic.bytes == b"abc" && diagnostic.exceeded + error, + GitInventoryError::OutputBound { + operation: "test diagnostics", + stream: "diagnostic bytes", + maximum: GIT_DIAGNOSTIC_LIMIT_BYTES, + .. + } )); } #[test] -fn simultaneous_git_failures_preserve_the_detected_error() { - let primary = GitInventoryError::OutputFraming { - operation: "test inventory", - }; - let cleanup = GitInventoryError::Run { - operation: "test inventory", - action: "stop", - source: io::Error::other("cleanup failed"), - }; - - let error = preserve_error(primary, Err(cleanup)); +fn stalled_git_process_obeys_the_inventory_deadline() -> Result<(), Box> { + let executable = env::current_exe()?; + let child = Command::new(executable) + .args(["--exact", PARKED_CHILD_TEST]) + .env(PARKED_CHILD, "1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn()?; + let deadline = Duration::from_millis(25); + + let result = paths_with_deadline(&[], "test Git inventory", deadline, move |_command| { + Ok(child) + }); assert!(matches!( - error, - GitInventoryError::Cleanup { - primary, - cleanup, - } if matches!(*primary, GitInventoryError::OutputFraming { - operation: "test inventory", - }) && matches!(*cleanup, GitInventoryError::Run { - operation: "test inventory", - action: "stop", - .. - }) + result, + Err(GitInventoryError::Process { + operation: "test Git inventory", + source: ProcessError::Timeout { + program: "git", + duration, + }, + }) if duration == deadline )); + Ok(()) +} + +#[test] +fn process_child_parks_indefinitely() { + if env::var_os(PARKED_CHILD).is_none() { + return; + } + loop { + std::thread::park(); + } } diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index e84abe7..1e8a003 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -3,6 +3,7 @@ const RUST_STANDARDS: &str = include_str!("../../docs/Rust Standards.md"); const BOUNDED_PROCESS: &str = include_str!("../src/bounded_process.rs"); const BOUNDED_PROCESS_CAPTURE: &str = include_str!("../src/bounded_process/capture.rs"); +const BOUNDED_PROCESS_CAPTURE_LIMIT: &str = include_str!("../src/bounded_process/capture_limit.rs"); const BOUNDED_PROCESS_ERROR: &str = include_str!("../src/bounded_process/error.rs"); const BOUNDED_PROCESS_INTERRUPT: &str = include_str!("../src/bounded_process/interrupt.rs"); const BOUNDED_PROCESS_GROUP: &str = include_str!("../src/bounded_process/process_group.rs"); @@ -41,6 +42,12 @@ fn source_scan_revalidates_repository_identity_after_reading() { ); } +#[test] +fn git_inventory_uses_the_deadline_bounded_process_layer() { + assert!(GIT_PROCESS.contains("const GIT_DEADLINE: Duration")); + assert!(GIT_PROCESS.contains("bounded_process::capture_with_limits(")); +} + #[test] fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); @@ -68,7 +75,21 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() BOUNDED_PROCESS, &["pub(crate) struct ProcessOutput", "pub(crate) fn status("], )?; - require_docs(BOUNDED_PROCESS_CAPTURE, &["pub(crate) fn capture("])?; + require_docs( + BOUNDED_PROCESS_CAPTURE, + &[ + "pub(crate) fn capture(", + "pub(crate) fn capture_with(", + "pub(crate) fn capture_with_limits(", + ], + )?; + require_docs( + BOUNDED_PROCESS_CAPTURE_LIMIT, + &[ + "pub(crate) struct CaptureLimits", + " pub(crate) const fn new(", + ], + )?; require_docs( BOUNDED_PROCESS_ERROR, &[ @@ -116,16 +137,14 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " Bytes,", " Items,", "pub(crate) enum GitInventoryError", - " Cleanup {", " DuplicatePath(", " EmptyPath {", " Failed {", " DiagnosticEncoding {", " OutputBound {", " OutputFraming {", - " Pipe {", + " Process {", " Run {", - " Worker {", ], )?; require_docs( @@ -147,7 +166,8 @@ fn require_docs(source: &str, declarations: &[&str]) -> Result<(), String> { .ok_or_else(|| format!("missing declaration `{declaration}`"))?; let documented = before .lines() - .next_back() + .rev() + .find(|line| !line.trim_start().starts_with("#[")) .is_some_and(|line| line.trim_start().starts_with("///")); if !documented { return Err(format!("missing rustdoc for `{declaration}`")); From 8957219196d2844135a7634bd280fca2158f4671 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:37:47 -0700 Subject: [PATCH 076/113] Fix: preserve process group ownership through capture --- CHANGELOG.md | 4 ++- ...007-terminal-signal-process-group-guard.md | 14 ++++++-- xtask/src/bounded_process/capture.rs | 13 +++---- xtask/tests/source_policy_contract.rs | 34 +++++++++++++++++++ 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53fbdec..ed87cc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,9 @@ after its public API and format compatibility policies are established. so transient replacement of the ambient repository path cannot redirect validation. Terminal signals now become typed refusals while an external repository task is active, so captured and inherited child groups are killed - and reaped before `xtask` returns. + and reaped before `xtask` returns. Captured-output readers finish while the + process-group leader remains waitable, and no cleanup path can address its + numeric group identity after the child has been reaped. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/docs/adr/0007-terminal-signal-process-group-guard.md b/docs/adr/0007-terminal-signal-process-group-guard.md index 4ba8cfb..a9a2e2b 100644 --- a/docs/adr/0007-terminal-signal-process-group-guard.md +++ b/docs/adr/0007-terminal-signal-process-group-guard.md @@ -37,9 +37,17 @@ per-stream capture limits. An observed terminal signal becomes a typed `ProcessError::Interrupted` refusal. The normal failure path then sends `SIGKILL` to the dedicated child process group, kills and reaps the child, and joins captured-output readers -before returning. The terminal signal is not sent directly to the child group; -one cleanup authority avoids races between signal delivery and mandatory -process-group termination. +before returning. Captured processes collect and join both readers before the +successful child wait consumes the process-group leader's waitable identity. +Every reader failure that can invoke group cleanup therefore occurs while that +identity is still owned, and no cleanup path addresses a numeric process-group +ID after the child has been reaped. Output-limit and late-interrupt refusals +found after a successful wait require no cleanup because the child and readers +have already terminated. + +The terminal signal is not sent directly to the child group; one cleanup +authority avoids races between signal delivery and mandatory process-group +termination. When no external operation is active, the guard restores the signal's default behavior. A signal observed while the final operation retires, or a second diff --git a/xtask/src/bounded_process/capture.rs b/xtask/src/bounded_process/capture.rs index 07d355c..a3f7ff5 100644 --- a/xtask/src/bounded_process/capture.rs +++ b/xtask/src/bounded_process/capture.rs @@ -121,10 +121,6 @@ impl CapturedProcess { program: &'static str, deadline: &ProcessDeadline, ) -> Result { - let status = match wait_for_child(program, &mut self.child, deadline, &self.interrupts) { - Ok(status) => status, - Err(error) => return Err(join_readers(self.stdout, self.stderr, error)), - }; let stdout = match self.stdout.receive(deadline, &self.interrupts) { Ok(output) => output, Err(error) => return Err(self.cleanup_readers(error)), @@ -140,11 +136,12 @@ impl CapturedProcess { if let Err(error) = self.stderr.join() { return Err(cleanup_process(&mut self.child, error)); } - refuse_exceeded(program, "stdout", self.limits.stdout_bytes(), &stdout) - .and_then(|()| refuse_exceeded(program, "stderr", self.limits.stderr_bytes(), &stderr)) - .map_err(|error| cleanup_process(&mut self.child, error))?; + let status = wait_for_child(program, &mut self.child, deadline, &self.interrupts)?; + refuse_exceeded(program, "stdout", self.limits.stdout_bytes(), &stdout).and_then(|()| { + refuse_exceeded(program, "stderr", self.limits.stderr_bytes(), &stderr) + })?; if let Some(error) = self.interrupts.refusal(program) { - return Err(cleanup_process(&mut self.child, error)); + return Err(error); } Ok(ProcessOutput { code: status.code(), diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 1e8a003..dcb7704 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -48,6 +48,40 @@ fn git_inventory_uses_the_deadline_bounded_process_layer() { assert!(GIT_PROCESS.contains("bounded_process::capture_with_limits(")); } +#[test] +fn captured_process_keeps_the_group_leader_until_reader_collection_finishes() +-> Result<(), &'static str> { + let (_, after_signature) = BOUNDED_PROCESS_CAPTURE + .split_once(" fn finish(") + .ok_or("captured process must retain its finish boundary")?; + let (body, _) = after_signature + .split_once("\n fn cleanup_readers") + .ok_or("captured process finish boundary must remain inspectable")?; + let wait = body + .find("wait_for_child") + .ok_or("captured process must reap its child")?; + + for operation in [ + "self.stdout.receive", + "self.stderr.receive", + "self.stdout.join", + "self.stderr.join", + ] { + let position = body + .find(operation) + .ok_or("captured process must collect and join both output streams")?; + assert!(position < wait, "child wait precedes {operation}"); + } + assert!( + !body + .get(wait..) + .unwrap_or_default() + .contains("cleanup_process"), + "cleanup may group-kill after the child ownership lifetime ends" + ); + Ok(()) +} + #[test] fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); From 1bde667152f79d5cf5f6b95a439da12b069554b1 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:40:38 -0700 Subject: [PATCH 077/113] Fix: synchronize descendant cleanup evidence --- CHANGELOG.md | 4 +- ...007-terminal-signal-process-group-guard.md | 4 +- .../bounded_process/process_group/tests.rs | 61 ++++++------------- xtask/tests/source_policy_contract.rs | 9 +++ 4 files changed, 34 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed87cc3..ec98100 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,9 @@ after its public API and format compatibility policies are established. repository task is active, so captured and inherited child groups are killed and reaped before `xtask` returns. Captured-output readers finish while the process-group leader remains waitable, and no cleanup path can address its - numeric group identity after the child has been reaped. + numeric group identity after the child has been reaped. Descendant cleanup + evidence now uses a pre-established socket disconnect instead of elapsed-time + reachability polling. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/docs/adr/0007-terminal-signal-process-group-guard.md b/docs/adr/0007-terminal-signal-process-group-guard.md index a9a2e2b..1b684ad 100644 --- a/docs/adr/0007-terminal-signal-process-group-guard.md +++ b/docs/adr/0007-terminal-signal-process-group-guard.md @@ -84,4 +84,6 @@ spawned, polled, killed, reaped, or read. The guard is private to `xtask`. It changes no Keep library API, content identity, durable format, or recovery protocol. Regression evidence sends `SIGINT` to a supervisor with an isolated descendant, requires the exact typed -refusal, and proves the descendant is no longer reachable. +refusal, and proves the descendant is no longer reachable. The test establishes +a witness socket before cleanup and waits for the kernel-reported disconnect; +elapsed time and scheduler latency do not classify descendant termination. diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 65bd24c..7e7e966 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -1,7 +1,7 @@ //! This module owns child process-group cleanup regression evidence. use std::env; -use std::io::{self, Write}; +use std::io; use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::Path; @@ -75,21 +75,18 @@ fn terminal_interrupt_terminates_the_isolated_descendant_group() .stderr(Stdio::null()) .spawn()?; wait_for_ready(&listener, &mut supervisor)?; + let descendant = UnixStream::connect(&socket)?; let supervisor_pid = Pid::from_raw(i32::try_from(supervisor.id())?).ok_or("supervisor process ID is zero")?; kill_process(supervisor_pid, Signal::INT)?; let status = wait_for_exit(&mut supervisor)?; - let descendant_survived = descendant_survived_cleanup(&socket)?; + require_descendant_disconnect(descendant)?; directory.close()?; assert!( status.success(), "interrupted supervisor did not exit cleanly: {status:?}" ); - assert!( - !descendant_survived, - "terminal interrupt left the isolated descendant reachable" - ); Ok(()) } @@ -143,6 +140,7 @@ fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box Result<(), Box Result { - let expires = Instant::now() - .checked_add(Duration::from_millis(500)) - .ok_or_else(|| io::Error::other("descendant cleanup deadline overflow"))?; - loop { - match UnixStream::connect(socket) { - Ok(mut stream) if Instant::now() >= expires => { - stream.write_all(b"x")?; - return Ok(true); - } - Ok(mut stream) => match stream.write_all(b"p") { - Ok(()) => thread::yield_now(), - Err(source) - if matches!( - source.kind(), - io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset - ) => - { - return Ok(false); - } - Err(source) => return Err(source), - }, - Err(source) - if matches!( - source.kind(), - io::ErrorKind::ConnectionRefused | io::ErrorKind::NotFound - ) => - { - return Ok(false); - } - Err(source) => return Err(source), +fn require_descendant_disconnect(mut descendant: UnixStream) -> Result<(), io::Error> { + let mut byte = [0_u8; 1]; + match io::Read::read_exact(&mut descendant, &mut byte) { + Err(source) + if matches!( + source.kind(), + io::ErrorKind::UnexpectedEof | io::ErrorKind::ConnectionReset + ) => + { + Ok(()) } + Err(source) => Err(source), + Ok(()) => Err(io::Error::other( + "terminated descendant unexpectedly wrote to its witness socket", + )), } } diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index dcb7704..7c60171 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -7,6 +7,8 @@ const BOUNDED_PROCESS_CAPTURE_LIMIT: &str = include_str!("../src/bounded_process const BOUNDED_PROCESS_ERROR: &str = include_str!("../src/bounded_process/error.rs"); const BOUNDED_PROCESS_INTERRUPT: &str = include_str!("../src/bounded_process/interrupt.rs"); const BOUNDED_PROCESS_GROUP: &str = include_str!("../src/bounded_process/process_group.rs"); +const BOUNDED_PROCESS_GROUP_TESTS: &str = + include_str!("../src/bounded_process/process_group/tests.rs"); const BOUNDED_PROCESS_READER: &str = include_str!("../src/bounded_process/reader.rs"); const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); @@ -82,6 +84,13 @@ fn captured_process_keeps_the_group_leader_until_reader_collection_finishes() Ok(()) } +#[test] +fn descendant_cleanup_uses_disconnect_evidence_instead_of_elapsed_time() { + assert!(!BOUNDED_PROCESS_GROUP_TESTS.contains("descendant_survived_cleanup")); + assert!(!BOUNDED_PROCESS_GROUP_TESTS.contains("Duration::from_millis(500)")); + assert!(BOUNDED_PROCESS_GROUP_TESTS.contains("require_descendant_disconnect")); +} + #[test] fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); From 28fb2d66c61b47b7f19a9cd301533e857b36ce1f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:43:58 -0700 Subject: [PATCH 078/113] Fix: bound documentation Git fixtures --- CHANGELOG.md | 20 ++++++++++--------- docs/dependencies/documentation-toolchain.md | 5 +++++ .../corpus/test_repository.rs | 20 +++++++++++++++---- xtask/tests/source_policy_contract.rs | 9 +++++++++ 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec98100..d9b1195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,15 +38,17 @@ after its public API and format compatibility policies are established. execution, and output collection. Git-backed process fixtures ignore system and global Git configuration and preserve non-UTF-8 template paths without lossy conversion. Documentation - Git inventory and tools start from one retained repository directory handle, - so transient replacement of the ambient repository path cannot redirect - validation. Terminal signals now become typed refusals while an external - repository task is active, so captured and inherited child groups are killed - and reaped before `xtask` returns. Captured-output readers finish while the - process-group leader remains waitable, and no cleanup path can address its - numeric group identity after the child has been reaped. Descendant cleanup - evidence now uses a pre-established socket disconnect instead of elapsed-time - reachability polling. + corpus fixture commands also use the bounded process authority with dedicated + groups, null output, and a two-minute deadline. Documentation Git inventory + and tools start from one retained repository directory handle, so transient + replacement of the ambient repository path cannot redirect validation. + Terminal signals now become typed refusals while an external repository task + is active, so captured and inherited child groups are killed and reaped + before `xtask` returns. Captured-output readers finish while the process-group + leader remains waitable, and no cleanup path can address its numeric group + identity after the child has been reaped. Descendant cleanup evidence now + uses a pre-established socket disconnect instead of elapsed-time reachability + polling. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index 04b74aa..dd0a30e 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -57,6 +57,11 @@ at most the 16 MiB path-stream bound, and diagnostics retain at most 64 KiB. A timeout, terminal signal, reader failure, or exceeded bound terminates the whole group and reaps the child before the task refuses. +Documentation corpus tests construct their Git fixtures through the same +bounded process authority. Each fixture command runs in a dedicated process +group with a two-minute deadline and null output streams, so a stalled command +or descendant cannot outlive the test boundary. + Git inventory and each validation tool start through one retained repository directory handle. Child-only setup changes directory through that handle after fork and before exec; the parent working directory does not change. Replacing diff --git a/xtask/src/documentation_integrity/corpus/test_repository.rs b/xtask/src/documentation_integrity/corpus/test_repository.rs index 2fd37bc..0bafb3c 100644 --- a/xtask/src/documentation_integrity/corpus/test_repository.rs +++ b/xtask/src/documentation_integrity/corpus/test_repository.rs @@ -2,20 +2,32 @@ use std::error::Error; use std::path::Path; -use std::process::Command; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::bounded_process; + +const GIT_FIXTURE_DEADLINE: Duration = Duration::from_mins(2); pub(in crate::documentation_integrity) fn run_git( root: &Path, arguments: &[&str], ) -> Result<(), Box> { - let output = Command::new("git") + let mut command = Command::new("git"); + command .args(arguments) .current_dir(root) .env("GIT_CONFIG_NOSYSTEM", "1") .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_COUNT", "0") - .output()?; - if output.status.success() { + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let output = bounded_process::status( + "documentation Git fixture", + &mut command, + Some(GIT_FIXTURE_DEADLINE), + )?; + if output.succeeded { Ok(()) } else { Err(format!("git fixture command failed: {arguments:?}").into()) diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 7c60171..c1ef119 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -11,6 +11,8 @@ const BOUNDED_PROCESS_GROUP_TESTS: &str = include_str!("../src/bounded_process/process_group/tests.rs"); const BOUNDED_PROCESS_READER: &str = include_str!("../src/bounded_process/reader.rs"); const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); +const DOCUMENTATION_TEST_REPOSITORY: &str = + include_str!("../src/documentation_integrity/corpus/test_repository.rs"); const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs"); const GIT_PROCESS: &str = include_str!("../src/git_inventory/process.rs"); @@ -91,6 +93,13 @@ fn descendant_cleanup_uses_disconnect_evidence_instead_of_elapsed_time() { assert!(BOUNDED_PROCESS_GROUP_TESTS.contains("require_descendant_disconnect")); } +#[test] +fn documentation_git_fixtures_use_the_bounded_process_layer() { + assert!(DOCUMENTATION_TEST_REPOSITORY.contains("bounded_process::status(")); + assert!(DOCUMENTATION_TEST_REPOSITORY.contains("GIT_FIXTURE_DEADLINE")); + assert!(!DOCUMENTATION_TEST_REPOSITORY.contains(".output()")); +} + #[test] fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); From a8ddc03b04ba0cb68af5baefbdb409c84870d580 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:45:15 -0700 Subject: [PATCH 079/113] Fix: isolate documentation Git fixtures --- CHANGELOG.md | 14 ++++++++------ docs/dependencies/documentation-toolchain.md | 6 +++++- .../corpus/test_repository.rs | 7 +++++++ xtask/tests/source_policy_contract.rs | 8 ++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b1195..ed93800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,12 +36,14 @@ after its public API and format compatibility policies are established. drift before and after each external tool, and applies a two-minute deadline across Git inventory, validation-tool execution, and output collection. - Git-backed process fixtures ignore system and global Git configuration and - preserve non-UTF-8 template paths without lossy conversion. Documentation - corpus fixture commands also use the bounded process authority with dedicated - groups, null output, and a two-minute deadline. Documentation Git inventory - and tools start from one retained repository directory handle, so transient - replacement of the ambient repository path cannot redirect validation. + Git-backed process fixtures clear the inherited environment, explicitly + admit the executable search path and `C` locale, ignore system and global Git + configuration, and preserve non-UTF-8 template paths without lossy + conversion. Documentation corpus fixture commands also use the bounded + process authority with dedicated groups, null output, and a two-minute + deadline. Documentation Git inventory and tools start from one retained + repository directory handle, so transient replacement of the ambient + repository path cannot redirect validation. Terminal signals now become typed refusals while an external repository task is active, so captured and inherited child groups are killed and reaped before `xtask` returns. Captured-output readers finish while the process-group diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index dd0a30e..e9a571a 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -60,7 +60,11 @@ whole group and reaps the child before the task refuses. Documentation corpus tests construct their Git fixtures through the same bounded process authority. Each fixture command runs in a dedicated process group with a two-minute deadline and null output streams, so a stalled command -or descendant cannot outlive the test boundary. +or descendant cannot outlive the test boundary. The helper clears the inherited +environment, then admits only the ambient executable search path, the `C` +locale, and explicit null system and global Git configuration. Ambient Git +directory, worktree, index, object, and configuration variables cannot redirect +a fixture. Git inventory and each validation tool start through one retained repository directory handle. Child-only setup changes directory through that handle after diff --git a/xtask/src/documentation_integrity/corpus/test_repository.rs b/xtask/src/documentation_integrity/corpus/test_repository.rs index 0bafb3c..756b367 100644 --- a/xtask/src/documentation_integrity/corpus/test_repository.rs +++ b/xtask/src/documentation_integrity/corpus/test_repository.rs @@ -1,6 +1,8 @@ //! This module owns hermetic Git commands for corpus regression repositories. +use std::env; use std::error::Error; +use std::io; use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; @@ -13,13 +15,18 @@ pub(in crate::documentation_integrity) fn run_git( root: &Path, arguments: &[&str], ) -> Result<(), Box> { + let path = env::var_os("PATH") + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "PATH is unavailable"))?; let mut command = Command::new("git"); command .args(arguments) .current_dir(root) + .env_clear() + .env("PATH", path) .env("GIT_CONFIG_NOSYSTEM", "1") .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_COUNT", "0") + .env("LC_ALL", "C") .stdout(Stdio::null()) .stderr(Stdio::null()); let output = bounded_process::status( diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index c1ef119..0dfece0 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -100,6 +100,14 @@ fn documentation_git_fixtures_use_the_bounded_process_layer() { assert!(!DOCUMENTATION_TEST_REPOSITORY.contains(".output()")); } +#[test] +fn documentation_git_fixtures_clear_the_ambient_environment() { + assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env_clear()")); + assert!(DOCUMENTATION_TEST_REPOSITORY.contains("env::var_os(\"PATH\")")); + assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env(\"PATH\"")); + assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env(\"LC_ALL\", \"C\")")); +} + #[test] fn process_fixtures_do_not_write_to_rust_stdout() { assert!(!BOUNDED_PROCESS_TESTS.contains("io::stdout()")); From 1e13a4e53957a28e5ee8199ac262b0a9a6199a2d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:46:07 -0700 Subject: [PATCH 080/113] Fix: document documentation Git fixtures --- .../src/documentation_integrity/corpus/test_repository.rs | 8 ++++++++ xtask/tests/source_policy_contract.rs | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/xtask/src/documentation_integrity/corpus/test_repository.rs b/xtask/src/documentation_integrity/corpus/test_repository.rs index 756b367..2b8c9d9 100644 --- a/xtask/src/documentation_integrity/corpus/test_repository.rs +++ b/xtask/src/documentation_integrity/corpus/test_repository.rs @@ -11,6 +11,14 @@ use crate::bounded_process; const GIT_FIXTURE_DEADLINE: Duration = Duration::from_mins(2); +/// Runs one Git fixture command under the repository's bounded process policy. +/// +/// The call blocks for at most two minutes, starts Git in a dedicated process +/// group, discards both output streams, and terminates descendants on timeout +/// or interruption. The child inherits only the admitted executable search +/// path, deterministic locale, and null system and global Git configuration. +/// Process failures retain their typed source; a nonzero Git status reports the +/// attempted arguments without admitting tool output. pub(in crate::documentation_integrity) fn run_git( root: &Path, arguments: &[&str], diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 0dfece0..5eae83b 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -190,6 +190,10 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " pub(super) fn join(", ], )?; + require_docs( + DOCUMENTATION_TEST_REPOSITORY, + &["pub(in crate::documentation_integrity) fn run_git("], + )?; require_docs( GIT_INVENTORY_ERROR, &[ From d59345d69ad72539a30dccbd15ea2148f864b582 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:49:53 -0700 Subject: [PATCH 081/113] Fix: require read-only workflow permissions --- CHANGELOG.md | 3 ++- docs/dependencies/documentation-toolchain.md | 3 +++ .../documentation_integrity/workflow_contract.rs | 9 +++++++++ .../workflow_contract/execution_context.rs | 14 ++++++++++++++ .../workflow_contract/tests.rs | 4 ++++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed93800..3bd31d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,8 @@ after its public API and format compatibility policies are established. unreviewed action steps, refuses alternate setup-node actions, requires the reviewed Node version, rejects unreviewed workflow/job run defaults and step execution fields, pins the - documentation runner and job deadline, rejects guarded or failure-tolerant + documentation runner and job deadline, requires the exact top-level + `contents: read` permission mapping, rejects guarded or failure-tolerant documentation jobs and required steps, requires each Dependabot update block to choose exactly one directory field form, and requires the documentation workflow to run for pushes to `main` and every pull request, diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index e9a571a..814c532 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -109,6 +109,9 @@ The documentation job has read-only repository permissions and disables checkout credential persistence. It writes tools only beneath `RUNNER_TEMP`. A failed or interrupted installation leaves no authoritative state and requires no recovery; a subsequent job starts from a fresh runner. +The Rust workflow contract requires the exact top-level `contents: read` +permission mapping and refuses `write-all`, write authority, additional scopes, +or omitted permissions before admitting any documentation step. ## Review triggers diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 0d02616..17fae2d 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -58,6 +58,9 @@ fn admitted_steps(workflow: &str) -> Result, Documentatio "workflow runs on reviewed push and pull request triggers", )); } + if !permissions_are_read_only(document) { + return Err(contract("workflow permissions are read-only")); + } reviewed_steps(documentation_steps(document)?) } @@ -71,6 +74,12 @@ fn triggers_are_reviewed(document: &Yaml) -> bool { && triggers["pull_request"].is_null() } +fn permissions_are_read_only(document: &Yaml) -> bool { + let permissions = &document["permissions"]; + mapping_has_exact_fields(permissions, &["contents"]) + && permissions["contents"].as_str() == Some("read") +} + fn documentation_steps(document: &Yaml) -> Result<&Vec, DocumentationError> { if !document["defaults"].is_badvalue() || !document["env"].is_badvalue() { return Err(contract( diff --git a/xtask/src/documentation_integrity/workflow_contract/execution_context.rs b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs index 902a50e..0ede2bf 100644 --- a/xtask/src/documentation_integrity/workflow_contract/execution_context.rs +++ b/xtask/src/documentation_integrity/workflow_contract/execution_context.rs @@ -26,6 +26,20 @@ fn workflow_run_defaults_cannot_replace_required_commands() { ); } +#[test] +fn workflow_cannot_grant_write_all_permissions() { + let workflow = WORKFLOW.replace("permissions:\n contents: read", "permissions: write-all"); + + assert_contract(&workflow, "workflow permissions are read-only"); +} + +#[test] +fn workflow_content_permission_cannot_grant_write_authority() { + let workflow = WORKFLOW.replace(" contents: read", " contents: write"); + + assert_contract(&workflow, "workflow permissions are read-only"); +} + #[test] fn job_run_defaults_cannot_replace_required_commands() { let workflow = WORKFLOW.replace( diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index 8666822..cd5d4bc 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -11,6 +11,8 @@ on: push: branches: [main] pull_request: +permissions: + contents: read jobs: documentation: name: Documentation and workflow integrity @@ -111,6 +113,8 @@ on: push: branches: [main] pull_request: +permissions: + contents: read jobs: documentation: name: Documentation and workflow integrity From 2986a7b8207c602dfb887b664e5d87b4fe30cf86 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 18:58:59 -0700 Subject: [PATCH 082/113] Fix: make malformed input checks executable --- .github/workflows/ci.yml | 4 +- CHANGELOG.md | 3 + docs/Documentation Standards.md | 17 +- docs/dependencies/documentation-toolchain.md | 4 + xtask/src/documentation_integrity.rs | 5 + xtask/src/documentation_integrity/error.rs | 23 ++- .../documentation_integrity/error/display.rs | 16 ++ .../src/documentation_integrity/execution.rs | 9 +- .../execution/external_tests.rs | 84 --------- .../execution/refusal_check.rs | 177 ++++++++++++++++++ .../workflow_contract/reviewed_step.rs | 3 +- .../workflow_contract/tests.rs | 4 +- xtask/src/main.rs | 6 +- xtask/src/task_error.rs | 3 +- xtask/src/test_directory.rs | 4 + xtask/tests/cli_contract.rs | 3 +- 16 files changed, 255 insertions(+), 110 deletions(-) delete mode 100644 xtask/src/documentation_integrity/execution/external_tests.rs create mode 100644 xtask/src/documentation_integrity/execution/refusal_check.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c3c3ca..0379ac7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,9 +99,7 @@ jobs: "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" - name: Verify malformed-input refusal laws - run: | - cargo test --locked --package xtask \ - documentation_integrity::execution::external_tests -- --ignored + run: cargo xtask documentation-refusal-check - name: Check documentation and workflows run: cargo xtask documentation-integrity-check diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bd31d1..acd2694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,9 @@ after its public API and format compatibility policies are established. documentation jobs and required steps, requires each Dependabot update block to choose exactly one directory field form, and requires the documentation workflow to run for pushes to `main` and every pull request, + executes malformed Markdown and workflow evidence through the named + `cargo xtask documentation-refusal-check` boundary instead of a + zero-match-successful libtest substring filter, bounds each admitted documentation source to 4 MiB and each selected corpus to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time diff --git a/docs/Documentation Standards.md b/docs/Documentation Standards.md index 4a3afea..da5cdc8 100644 --- a/docs/Documentation Standards.md +++ b/docs/Documentation Standards.md @@ -553,13 +553,16 @@ The same Rust command checks workflows with `actionlint` 1.7.12 and refuses another version. It also verifies the committed Node lock graph, Dependabot manifest coverage, and the documentation job's delegation to this command. The dedicated `documentation` job in `.github/workflows/ci.yml` installs the -pinned tools, runs malformed-input refusal laws and the repository-owned -command, and verifies repository whitespace before admitting the result as CI -evidence. The Rust workflow contract pins that job to `ubuntu-latest` with a -ten-minute deadline, rejects workflow and job run defaults, and admits only the -reviewed action and command step fields. A custom shell, working directory, -environment, or other unreviewed execution modifier cannot impersonate a -required command. +pinned tools, runs `cargo xtask documentation-refusal-check`, runs the +repository-owned integrity command, and verifies repository whitespace before +admitting the result as CI evidence. The named refusal command constructs and +executes both malformed-input scenarios directly, so removing or renaming +either scenario breaks the command instead of producing a successful +zero-test result. The Rust workflow contract pins that job to `ubuntu-latest` +with a ten-minute deadline, rejects workflow and job run defaults, and admits +only the reviewed action and command step fields. A custom shell, working +directory, environment, or other unreviewed execution modifier cannot +impersonate a required command. CI SHOULD block on facts it can determine reliably: diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index 814c532..7fe28ff 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -112,6 +112,10 @@ state and requires no recovery; a subsequent job starts from a fresh runner. The Rust workflow contract requires the exact top-level `contents: read` permission mapping and refuses `write-all`, write authority, additional scopes, or omitted permissions before admitting any documentation step. +The job invokes `cargo xtask documentation-refusal-check` after installing the +pinned tools. That named Rust command directly executes the broken-fragment and +invalid-workflow scenarios and requires their exact tool refusals; it does not +depend on a libtest substring filter or accept a zero-test run. ## Review triggers diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index 508aa2d..eb6d255 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -16,6 +16,11 @@ use crate::repository_file::RepositoryRoot; pub(super) use error::DocumentationError; +/// Runs the pinned-tool malformed-input refusal evidence. +pub(super) fn check_refusals() -> Result<(), DocumentationError> { + execution::check_refusals() +} + pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { let repository_root = RepositoryRoot::open(repository_path).map_err(|source| { DocumentationError::RepositoryRootInspect { diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 9a484de..ec9c96e 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -52,6 +52,16 @@ pub(crate) enum DocumentationError { source: FromUtf8Error, }, Process(ProcessError), + /// A filesystem operation could not construct or remove a refusal fixture. + RefusalFixture { + action: &'static str, + source: io::Error, + }, + /// A malformed-input scenario did not produce its exact reviewed refusal. + RefusalMismatch { + scenario: &'static str, + observed: Option>, + }, RepositoryFileEncoding { path: &'static str, source: FromUtf8Error, @@ -128,15 +138,19 @@ impl Error for DocumentationError { Self::CheckFailures { first, .. } => Some(first), Self::GitInventory(error) => Some(error), Self::Process(error) => Some(error), - Self::Inspect { source, .. } | Self::RepositoryFileInspect { source, .. } => { - Some(source) - } + Self::Inspect { source, .. } + | Self::RefusalFixture { source, .. } + | Self::RepositoryFileInspect { source, .. } + | Self::RepositoryRootInspect { source, .. } => Some(source), Self::PathEncoding { source, .. } | Self::RepositoryFileEncoding { source, .. } => { Some(source) } + Self::RefusalMismatch { + observed: Some(error), + .. + } => Some(error), Self::RepositoryJson { source, .. } => Some(source), Self::RepositoryYaml { source, .. } => Some(source), - Self::RepositoryRootInspect { source, .. } => Some(source), Self::ToolOutputEncoding { source, .. } => Some(source), Self::ToolUnavailable { source, .. } => Some(source), Self::CorpusFileTooLarge { .. } @@ -146,6 +160,7 @@ impl Error for DocumentationError { | Self::EmptyCorpus(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } + | Self::RefusalMismatch { observed: None, .. } | Self::RepositoryFileNonRegular(_) | Self::RepositoryFileTooLarge { .. } | Self::RepositoryContract { .. } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 826828b..9573835 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -45,6 +45,22 @@ impl fmt::Display for DocumentationError { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } Self::Process(error) => write!(formatter, "{error}"), + Self::RefusalFixture { action, .. } => { + write!( + formatter, + "cannot {action} for documentation refusal evidence" + ) + } + Self::RefusalMismatch { scenario, observed } => { + write!( + formatter, + "documentation refusal scenario `{scenario}` did not produce its reviewed error" + )?; + if let Some(error) = observed { + write!(formatter, ": {error}")?; + } + Ok(()) + } error @ (Self::RepositoryFileEncoding { .. } | Self::RepositoryFileInspect { .. } | Self::RepositoryFileNonRegular(_) diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index f4b5e15..2f2bba1 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -1,6 +1,7 @@ //! This module owns bounded execution of admitted documentation tools. mod corpus_guard; +mod refusal_check; use std::process::{Command, Stdio}; use std::time::Duration; @@ -39,6 +40,11 @@ pub(super) fn run( run_with(&mut runner, markdown.paths(), workflows.paths()) } +/// Executes both named malformed-input scenarios through the production runner. +pub(super) fn check_refusals() -> Result<(), DocumentationError> { + refusal_check::check() +} + fn run_with( runner: &mut impl ToolRunner, markdown: &[String], @@ -168,9 +174,6 @@ impl ToolRunner for ExternalToolRunner<'_> { } } -#[cfg(test)] -#[path = "execution/external_tests.rs"] -mod external_tests; #[cfg(test)] #[path = "execution/tests.rs"] mod tests; diff --git a/xtask/src/documentation_integrity/execution/external_tests.rs b/xtask/src/documentation_integrity/execution/external_tests.rs deleted file mode 100644 index cc8708f..0000000 --- a/xtask/src/documentation_integrity/execution/external_tests.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! This module owns pinned-tool malformed-input refusal evidence. - -use std::fs; - -use crate::repository_file::RepositoryRoot; -use crate::test_directory::TestDirectory; - -use super::{DocumentationError, DocumentationTool, ExternalToolRunner}; - -#[test] -#[ignore = "requires pinned documentation tools installed by the documentation CI job"] -fn broken_internal_fragment_is_refused() -> Result<(), Box> { - let directory = TestDirectory::create("broken-fragment")?; - fs::write( - directory.path().join("source.md"), - "# Source\n\n[Missing](target.md#missing-heading)\n", - )?; - fs::write(directory.path().join("target.md"), "# Present heading\n")?; - let repository_root = RepositoryRoot::open(directory.path())?; - let process_directory = repository_root.process_directory()?; - let refusal = { - let mut runner = ExternalToolRunner { - process_directory: &process_directory, - }; - super::admit_version(&mut runner, DocumentationTool::Markdownlint)?; - super::admit_version(&mut runner, DocumentationTool::Lychee)?; - super::run_check( - &mut runner, - DocumentationTool::Markdownlint, - &[String::from("source.md"), String::from("target.md")], - )?; - super::run_check( - &mut runner, - DocumentationTool::Lychee, - &[String::from("source.md"), String::from("target.md")], - ) - }; - assert!(matches!( - refusal, - Err(DocumentationError::ToolFailed { - program: "lychee", - code: Some(2), - ref stdout, - ref stderr, - }) if format!("{stdout}\n{stderr}").contains("Cannot find fragment") - )); - directory.close()?; - Ok(()) -} - -#[test] -#[ignore = "requires pinned documentation tools installed by the documentation CI job"] -fn invalid_workflow_is_refused() -> Result<(), Box> { - let directory = TestDirectory::create("invalid-workflow")?; - fs::create_dir_all(directory.path().join(".github/workflows"))?; - fs::write( - directory.path().join(".github/workflows/invalid.yml"), - "name: Invalid\non: [push\n", - )?; - let repository_root = RepositoryRoot::open(directory.path())?; - let process_directory = repository_root.process_directory()?; - let refusal = { - let mut runner = ExternalToolRunner { - process_directory: &process_directory, - }; - super::admit_version(&mut runner, DocumentationTool::Actionlint)?; - super::run_check( - &mut runner, - DocumentationTool::Actionlint, - &[String::from(".github/workflows/invalid.yml")], - ) - }; - assert!(matches!( - refusal, - Err(DocumentationError::ToolFailed { - program: "actionlint", - code: Some(1), - ref stdout, - ref stderr, - }) if format!("{stdout}\n{stderr}").contains("could not parse as YAML") - )); - directory.close()?; - Ok(()) -} diff --git a/xtask/src/documentation_integrity/execution/refusal_check.rs b/xtask/src/documentation_integrity/execution/refusal_check.rs new file mode 100644 index 0000000..c098086 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/refusal_check.rs @@ -0,0 +1,177 @@ +//! This module owns executable pinned-tool malformed-input refusal evidence. + +use std::fs; +use std::io; + +use crate::documentation_integrity::DocumentationError; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::{DocumentationTool, ExternalToolRunner}; + +/// Requires exact pinned-tool refusals for every malformed-input scenario. +pub(super) fn check() -> Result<(), DocumentationError> { + broken_internal_fragment_is_refused()?; + invalid_workflow_is_refused() +} + +fn broken_internal_fragment_is_refused() -> Result<(), DocumentationError> { + let directory = TestDirectory::create("broken-fragment") + .map_err(|source| fixture_error("create broken-fragment directory", source))?; + fs::write( + directory.path().join("source.md"), + "# Source\n\n[Missing](target.md#missing-heading)\n", + ) + .map_err(|source| fixture_error("write broken-fragment source", source))?; + fs::write(directory.path().join("target.md"), "# Present heading\n") + .map_err(|source| fixture_error("write broken-fragment target", source))?; + let repository_root = RepositoryRoot::open(directory.path()) + .map_err(|source| fixture_error("open broken-fragment repository", source))?; + let process_directory = repository_root + .process_directory() + .map_err(|source| fixture_error("open broken-fragment process directory", source))?; + let refusal = { + let mut runner = ExternalToolRunner { + process_directory: &process_directory, + }; + super::admit_version(&mut runner, DocumentationTool::Markdownlint)?; + super::admit_version(&mut runner, DocumentationTool::Lychee)?; + super::run_check( + &mut runner, + DocumentationTool::Markdownlint, + &[String::from("source.md"), String::from("target.md")], + )?; + super::run_check( + &mut runner, + DocumentationTool::Lychee, + &[String::from("source.md"), String::from("target.md")], + ) + }; + require_refusal( + refusal, + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + )?; + directory + .close() + .map_err(|source| fixture_error("remove broken-fragment directory", source))?; + Ok(()) +} + +fn invalid_workflow_is_refused() -> Result<(), DocumentationError> { + let directory = TestDirectory::create("invalid-workflow") + .map_err(|source| fixture_error("create invalid-workflow directory", source))?; + fs::create_dir_all(directory.path().join(".github/workflows")) + .map_err(|source| fixture_error("create invalid workflow directory", source))?; + fs::write( + directory.path().join(".github/workflows/invalid.yml"), + "name: Invalid\non: [push\n", + ) + .map_err(|source| fixture_error("write invalid workflow", source))?; + let repository_root = RepositoryRoot::open(directory.path()) + .map_err(|source| fixture_error("open invalid-workflow repository", source))?; + let process_directory = repository_root + .process_directory() + .map_err(|source| fixture_error("open invalid-workflow process directory", source))?; + let refusal = { + let mut runner = ExternalToolRunner { + process_directory: &process_directory, + }; + super::admit_version(&mut runner, DocumentationTool::Actionlint)?; + super::run_check( + &mut runner, + DocumentationTool::Actionlint, + &[String::from(".github/workflows/invalid.yml")], + ) + }; + require_refusal( + refusal, + "invalid workflow", + "actionlint", + Some(1), + "could not parse as YAML", + )?; + directory + .close() + .map_err(|source| fixture_error("remove invalid-workflow directory", source))?; + Ok(()) +} + +fn require_refusal( + refusal: Result<(), DocumentationError>, + scenario: &'static str, + program: &'static str, + code: Option, + diagnostic: &str, +) -> Result<(), DocumentationError> { + match refusal { + Err(DocumentationError::ToolFailed { + program: observed_program, + code: observed_code, + stdout, + stderr, + }) if observed_program == program + && observed_code == code + && format!("{stdout}\n{stderr}").contains(diagnostic) => + { + Ok(()) + } + Err(observed) => Err(DocumentationError::RefusalMismatch { + scenario, + observed: Some(Box::new(observed)), + }), + Ok(()) => Err(DocumentationError::RefusalMismatch { + scenario, + observed: None, + }), + } +} + +const fn fixture_error(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::RefusalFixture { action, source } +} + +#[cfg(test)] +mod tests { + use super::{DocumentationError, require_refusal}; + + #[test] + fn exact_tool_failure_is_executable_refusal_evidence() { + let refusal = Err(DocumentationError::ToolFailed { + program: "lychee", + code: Some(2), + stdout: String::from("Cannot find fragment"), + stderr: String::new(), + }); + + assert!( + require_refusal( + refusal, + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + ) + .is_ok() + ); + } + + #[test] + fn successful_malformed_input_is_a_typed_evidence_failure() { + assert!(matches!( + require_refusal( + Ok(()), + "broken internal fragment", + "lychee", + Some(2), + "Cannot find fragment", + ), + Err(DocumentationError::RefusalMismatch { + scenario: "broken internal fragment", + observed: None, + }) + )); + } +} diff --git a/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs index 2199fad..0023d62 100644 --- a/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs +++ b/xtask/src/documentation_integrity/workflow_contract/reviewed_step.rs @@ -1,7 +1,6 @@ //! This module owns the exact documentation-job step sequence. -const MALFORMED_INPUT_COMMAND: &str = r"cargo test --locked --package xtask \ - documentation_integrity::execution::external_tests -- --ignored"; +const MALFORMED_INPUT_COMMAND: &str = "cargo xtask documentation-refusal-check"; const INSTALL_TOOLS_COMMAND: &str = r#"documentation_tools="$RUNNER_TEMP/documentation-tools" scripts/install_documentation_tools.sh "$documentation_tools" printf '%s\n' \ diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index cd5d4bc..ca51fa4 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -37,9 +37,7 @@ jobs: "$documentation_tools/bin" \ "$documentation_tools/npm/node_modules/.bin" >> "$GITHUB_PATH" - name: Verify malformed inputs - run: | - cargo test --locked --package xtask \ - documentation_integrity::execution::external_tests -- --ignored + run: cargo xtask documentation-refusal-check - name: Verify run: cargo xtask documentation-integrity-check - name: Check whitespace diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 174057c..c167b41 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -64,10 +64,9 @@ mod source_structure; reason = "the parent command dispatcher is the only consumer" )] mod task_error; -#[cfg(test)] #[allow( clippy::redundant_pub_crate, - reason = "scoped test directories are shared by sibling test modules" + reason = "scoped evidence directories are shared by verification and test modules" )] mod test_directory; @@ -112,6 +111,9 @@ fn run(mut arguments: impl Iterator) -> Result<(), TaskError> { "documentation-integrity-check" => { documentation_integrity::check(repository_root)?; } + "documentation-refusal-check" => { + documentation_integrity::check_refusals()?; + } "prepare-fuzz-corpus" => { fuzz_seed_corpus::prepare(repository_root)?; } diff --git a/xtask/src/task_error.rs b/xtask/src/task_error.rs index 297a4f5..2df59ae 100644 --- a/xtask/src/task_error.rs +++ b/xtask/src/task_error.rs @@ -65,7 +65,8 @@ impl fmt::Display for TaskError { "usage: cargo xtask \ ", ), diff --git a/xtask/src/test_directory.rs b/xtask/src/test_directory.rs index 5d5f73d..d0bdf05 100644 --- a/xtask/src/test_directory.rs +++ b/xtask/src/test_directory.rs @@ -8,12 +8,14 @@ use std::sync::atomic::{AtomicU64, Ordering}; const CREATION_ATTEMPTS: u16 = 1_024; static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); +/// A collision-resistant temporary directory removed at the explicit boundary. pub(crate) struct TestDirectory { path: PathBuf, active: bool, } impl TestDirectory { + /// Creates one unique directory beneath the platform temporary directory. pub(crate) fn create(label: &str) -> Result { for _ in 0_u16..CREATION_ATTEMPTS { let sequence = next_sequence()?; @@ -31,10 +33,12 @@ impl TestDirectory { )) } + /// Returns the exact directory path owned by this scope. pub(crate) fn path(&self) -> &Path { &self.path } + /// Removes the exact owned directory tree and disables drop fallback. pub(crate) fn close(mut self) -> Result<(), io::Error> { let removal = match fs::remove_dir_all(&self.path) { Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), diff --git a/xtask/tests/cli_contract.rs b/xtask/tests/cli_contract.rs index 528ed7e..4be148d 100644 --- a/xtask/tests/cli_contract.rs +++ b/xtask/tests/cli_contract.rs @@ -89,7 +89,8 @@ fn missing_command_returns_the_versioned_usage_contract() -> Result<(), io::Erro b"Error: usage: cargo xtask \ \n" ); From 2185eee83f27db347ccdeb45f5d9e2dce33a1402 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:13:01 -0700 Subject: [PATCH 083/113] Fix: bind source checks to one file identity --- CHANGELOG.md | 6 +- docs/Rust Standards.md | 7 +- .../corpus/source_witness.rs | 57 +++----- xtask/src/repository_file.rs | 36 +++++ xtask/src/source_structure.rs | 52 +++---- xtask/src/source_structure/python_source.rs | 80 ++++------ xtask/src/source_structure/source_error.rs | 7 + xtask/src/source_structure/source_file.rs | 125 ++++++++++++++++ xtask/src/source_structure/tests.rs | 123 +--------------- .../src/source_structure/tests/replacement.rs | 137 ++++++++++++++++++ 10 files changed, 396 insertions(+), 234 deletions(-) create mode 100644 xtask/src/source_structure/source_file.rs create mode 100644 xtask/src/source_structure/tests/replacement.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index acd2694..389f648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,7 +90,11 @@ after its public API and format compatibility policies are established. non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment shebangs parse options, assignments, quoting, and split strings before classifying only the selected utility, so later command arguments cannot - impersonate Python and unresolved utility substitutions fail closed. + impersonate Python and unresolved utility substitutions fail closed. Source + execution, shebang, and physical-line evidence now come from one admitted + file descriptor whose identity is revalidated after each read phase, so path + replacement or in-place mutation cannot splice different file states into + one verification result. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 31e85d2..3be3ada 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -478,7 +478,12 @@ Hard CI limits: Count physical lines for deterministic enforcement. Blank and comment lines remain part of the maintainability surface; reviewers should also examine logical structure. The hard maximum applies to every executable source file -regardless of its filename suffix. +regardless of its filename suffix. Source classification, executable-shebang +inspection, and line counting MUST use one admitted file descriptor; reopening +the pathname between checks would permit replacement to splice evidence from +different files. The scanner MUST revalidate both the descriptor identity and +the current repository path after each read phase, and refuse concurrent +replacement or in-place mutation. A file above 300 lines MUST begin with a decomposition issue or contain an approved exception explaining why splitting it would damage locality. diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs index 0c76f22..e6ca187 100644 --- a/xtask/src/documentation_integrity/corpus/source_witness.rs +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -1,31 +1,19 @@ //! This module owns retained identity evidence for one documentation source. -use std::fs::{File, Metadata}; +use std::fs::File; use std::io; -use std::os::unix::fs::MetadataExt; use std::path::PathBuf; use super::CorpusKind; use crate::documentation_integrity::error::DocumentationError; -use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; pub(super) struct AdmittedSource { - identity: SourceIdentity, + identity: RepositoryFileIdentity, path: String, relative: PathBuf, } -#[derive(Eq, PartialEq)] -struct SourceIdentity { - device: u64, - inode: u64, - bytes: u64, - modified_seconds: i64, - modified_nanoseconds: i64, - changed_seconds: i64, - changed_nanoseconds: i64, -} - impl AdmittedSource { pub(super) fn admit( file: &File, @@ -33,16 +21,16 @@ impl AdmittedSource { relative: PathBuf, kind: CorpusKind, ) -> Result { - let metadata = metadata(file, kind, &path)?; + let identity = identity(file, kind, &path)?; Ok(Self { - identity: SourceIdentity::from(&metadata), + identity, path, relative, }) } pub(super) const fn bytes(&self) -> u64 { - self.identity.bytes + self.identity.bytes() } pub(super) fn path(&self) -> &str { @@ -72,7 +60,7 @@ impl AdmittedSource { return Err(changed(kind, &self.path)); } }; - let current = SourceIdentity::from(&metadata(¤t, kind, &self.path)?); + let current = identity(¤t, kind, &self.path)?; if current == self.identity { Ok(()) } else { @@ -81,27 +69,16 @@ impl AdmittedSource { } } -impl From<&Metadata> for SourceIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - bytes: metadata.len(), - modified_seconds: metadata.mtime(), - modified_nanoseconds: metadata.mtime_nsec(), - changed_seconds: metadata.ctime(), - changed_nanoseconds: metadata.ctime_nsec(), - } - } -} - -fn metadata(file: &File, kind: CorpusKind, path: &str) -> Result { - file.metadata() - .map_err(|source| DocumentationError::Inspect { - corpus: kind.label(), - path: path.to_owned(), - source, - }) +fn identity( + file: &File, + kind: CorpusKind, + path: &str, +) -> Result { + RepositoryFileIdentity::read(file).map_err(|source| DocumentationError::Inspect { + corpus: kind.label(), + path: path.to_owned(), + source, + }) } fn changed(kind: CorpusKind, path: &str) -> DocumentationError { diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs index 4ba135d..209d25c 100644 --- a/xtask/src/repository_file.rs +++ b/xtask/src/repository_file.rs @@ -8,6 +8,7 @@ use std::fs::File; use std::io; use std::os::fd::OwnedFd; +use std::os::unix::fs::MetadataExt as UnixMetadataExt; use std::path::{Path, PathBuf}; use std::process::{Child, Command}; @@ -57,11 +58,32 @@ struct DirectoryIdentity { inode: u64, } +#[derive(Eq, PartialEq)] +pub(crate) struct RepositoryFileIdentity { + device: u64, + inode: u64, + bytes: u64, + modified_seconds: i64, + modified_nanoseconds: i64, + changed_seconds: i64, + changed_nanoseconds: i64, +} + pub(crate) enum OpenRepositoryFileError { Io(io::Error), NonRegular, } +impl RepositoryFileIdentity { + pub(crate) fn read(file: &File) -> Result { + Ok(Self::from(&file.metadata()?)) + } + + pub(crate) const fn bytes(&self) -> u64 { + self.bytes + } +} + impl RepositoryRoot { pub(crate) fn open(path: &Path) -> Result { let directory = Dir::open_ambient_dir(path, ambient_authority())?; @@ -163,3 +185,17 @@ impl From<&cap_std::fs::Metadata> for DirectoryIdentity { } } } + +impl From<&std::fs::Metadata> for RepositoryFileIdentity { + fn from(metadata: &std::fs::Metadata) -> Self { + Self { + device: UnixMetadataExt::dev(metadata), + inode: UnixMetadataExt::ino(metadata), + bytes: metadata.len(), + modified_seconds: UnixMetadataExt::mtime(metadata), + modified_nanoseconds: UnixMetadataExt::mtime_nsec(metadata), + changed_seconds: UnixMetadataExt::ctime(metadata), + changed_nanoseconds: UnixMetadataExt::ctime_nsec(metadata), + } + } +} diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 2dcc706..c11fc6b 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -3,16 +3,17 @@ mod python_source; mod repository_path; mod source_error; +mod source_file; mod source_inventory; mod source_kind; use std::io::{self, BufRead, BufReader}; use std::path::Path; -use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; -use python_source::{FileExecution, refuse_executable_python}; +use crate::repository_file::RepositoryRoot; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; +use source_file::{AdmittedSource, FileExecution, SourceFileAdmission}; #[cfg(test)] use source_inventory::{ PRESENT_PATH_ARGUMENTS, select as select_source_inventory, select_source_paths, @@ -64,9 +65,13 @@ fn inventory_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in inventory.executable_candidates { - let execution = refuse_executable_python(source_root, relative.as_path())?; - if execution == FileExecution::Executable - && source_line_count(source_root, relative.as_path())? == SourceLineCount::Exceeded + let SourceFileAdmission::Regular(source) = + AdmittedSource::admit(source_root, relative.as_path())? + else { + continue; + }; + if source.execution() == FileExecution::Executable + && source_line_count(source_root, &source)? == SourceLineCount::Exceeded { violations.push(relative.as_path().to_owned()); } @@ -82,18 +87,19 @@ fn source_violations( ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { - let execution = refuse_executable_python(source_root, relative.as_path())?; - if execution == FileExecution::NonRegular { + let SourceFileAdmission::Regular(source) = + AdmittedSource::admit(source_root, relative.as_path())? + else { return Err(SourceStructureError::NonRegular( source_root.display_path(relative.as_path()), )); - } + }; if is_extensionless_file(relative.as_str().as_bytes()) - && execution == FileExecution::NonExecutable + && source.execution() == FileExecution::NonExecutable { continue; } - let lines = source_line_count(source_root, relative.as_path())?; + let lines = source_line_count(source_root, &source)?; if lines == SourceLineCount::Exceeded { violations.push(relative.as_path().to_owned()); } @@ -103,26 +109,16 @@ fn source_violations( fn source_line_count( source_root: &RepositoryRoot, - relative: &Path, -) -> Result { - source_line_count_with(source_root, relative, RepositoryRoot::open_file) -} - -fn source_line_count_with( - source_root: &RepositoryRoot, - relative: &Path, - open_source: impl FnOnce(&RepositoryRoot, &Path) -> Result, + source: &AdmittedSource, ) -> Result { - let path = source_root.display_path(relative); - let file = open_source(source_root, relative).map_err(|error| match error { - OpenRepositoryFileError::Io(source) => SourceStructureError::Inspect { - path: path.clone(), - source, - }, - OpenRepositoryFileError::NonRegular => SourceStructureError::NonRegular(path.clone()), + let lines = line_count(BufReader::new(source.file())).map_err(|error| { + SourceStructureError::Inspect { + path: source.path().to_owned(), + source: error, + } })?; - line_count(BufReader::new(file)) - .map_err(|source| SourceStructureError::Inspect { path, source }) + source.verify_current(source_root)?; + Ok(lines) } const fn exceeds_hard_limit(lines: u64) -> bool { diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 6ef8edf..863a319 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -3,63 +3,47 @@ mod environment; use std::fs::File; -use std::io::{self, Read}; -use std::os::unix::fs::PermissionsExt; +use std::io; +use std::os::unix::fs::FileExt; -use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +const SHEBANG_SCAN_BYTES: usize = 1_024; -use super::SourceStructureError; -use std::path::Path; - -const SHEBANG_SCAN_BYTES: u64 = 1_024; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum FileExecution { - Executable, - NonExecutable, - NonRegular, +pub(super) fn executable_uses_python(file: &File) -> Result { + let mut prefix = [0_u8; SHEBANG_SCAN_BYTES]; + let bytes = read_prefix(file, &mut prefix)?; + let admitted = prefix.get(..bytes).ok_or_else(prefix_bounds_error)?; + Ok(is_python_shebang(admitted)) } -pub(super) fn refuse_executable_python( - source_root: &RepositoryRoot, - relative: &Path, -) -> Result { - let path = source_root.display_path(relative); - let file = match source_root.open_file(relative) { - Ok(file) => file, - Err(OpenRepositoryFileError::NonRegular) => return Ok(FileExecution::NonRegular), - Err(OpenRepositoryFileError::Io(source)) => { - return Err(SourceStructureError::Inspect { path, source }); +fn read_prefix(file: &File, prefix: &mut [u8]) -> Result { + let mut filled = 0_usize; + while filled < prefix.len() { + let offset = u64::try_from(filled).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "shebang prefix offset exceeds u64", + ) + })?; + let remaining = prefix.get_mut(filled..).ok_or_else(prefix_bounds_error)?; + let read = match file.read_at(remaining, offset) { + Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, + result => result?, + }; + if read == 0 { + break; } - }; - let execution = file_execution(&file).map_err(|source| SourceStructureError::Inspect { - path: path.clone(), - source, - })?; - let python = execution == FileExecution::Executable - && executable_uses_python(file).map_err(|source| SourceStructureError::Inspect { - path: path.clone(), - source, + filled = filled.checked_add(read).ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "shebang prefix length overflow") })?; - if python { - Err(SourceStructureError::PythonSource(relative.to_owned())) - } else { - Ok(execution) - } -} - -fn file_execution(file: &File) -> Result { - if file.metadata()?.permissions().mode() & 0o111 == 0 { - Ok(FileExecution::NonExecutable) - } else { - Ok(FileExecution::Executable) } + Ok(filled) } -fn executable_uses_python(file: File) -> Result { - let mut prefix = Vec::new(); - file.take(SHEBANG_SCAN_BYTES).read_to_end(&mut prefix)?; - Ok(is_python_shebang(&prefix)) +fn prefix_bounds_error() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "shebang prefix bounds are inconsistent", + ) } fn is_python_shebang(prefix: &[u8]) -> bool { diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 8908996..2ca7570 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -23,6 +23,7 @@ pub(crate) enum SourceStructureError { NonRegular(PathBuf), PythonSource(PathBuf), RepositoryRootChanged(PathBuf), + SourceFileChanged(PathBuf), Violations { maximum: u64, paths: Vec, @@ -67,6 +68,11 @@ impl fmt::Display for SourceStructureError { escaped_path(formatter, path)?; formatter.write_str("`") } + Self::SourceFileChanged(path) => { + formatter.write_str("repository source changed during inspection: `")?; + escaped_path(formatter, path)?; + formatter.write_str("`") + } Self::Violations { maximum, paths } => violations_display(formatter, *maximum, paths), } } @@ -82,6 +88,7 @@ impl Error for SourceStructureError { | Self::NonRegular(_) | Self::PythonSource(_) | Self::RepositoryRootChanged(_) + | Self::SourceFileChanged(_) | Self::Violations { .. } => None, } } diff --git a/xtask/src/source_structure/source_file.rs b/xtask/src/source_structure/source_file.rs new file mode 100644 index 0000000..3123da3 --- /dev/null +++ b/xtask/src/source_structure/source_file.rs @@ -0,0 +1,125 @@ +//! This module owns one-handle admission of a repository source file. + +use std::fs::{File, Metadata}; +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; + +use super::SourceStructureError; +use super::python_source::executable_uses_python; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum FileExecution { + Executable, + NonExecutable, +} + +pub(super) enum SourceFileAdmission { + Regular(AdmittedSource), + NonRegular, +} + +pub(super) struct AdmittedSource { + execution: FileExecution, + file: File, + identity: RepositoryFileIdentity, + path: PathBuf, + relative: PathBuf, +} + +impl AdmittedSource { + pub(super) fn admit( + source_root: &RepositoryRoot, + relative: &Path, + ) -> Result { + let path = source_root.display_path(relative); + let file = match source_root.open_file(relative) { + Ok(file) => file, + Err(OpenRepositoryFileError::NonRegular) => { + return Ok(SourceFileAdmission::NonRegular); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(SourceStructureError::Inspect { path, source }); + } + }; + let metadata = file + .metadata() + .map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; + let execution = file_execution(&metadata); + let python = execution == FileExecution::Executable + && executable_uses_python(&file).map_err(|source| SourceStructureError::Inspect { + path: path.clone(), + source, + })?; + let source = Self { + execution, + file, + identity: RepositoryFileIdentity::from(&metadata), + path, + relative: relative.to_owned(), + }; + source.verify_current(source_root)?; + if python { + return Err(SourceStructureError::PythonSource(relative.to_owned())); + } + Ok(SourceFileAdmission::Regular(source)) + } + + pub(super) const fn execution(&self) -> FileExecution { + self.execution + } + + pub(super) const fn file(&self) -> &File { + &self.file + } + + pub(super) fn path(&self) -> &Path { + &self.path + } + + pub(super) fn verify_current( + &self, + source_root: &RepositoryRoot, + ) -> Result<(), SourceStructureError> { + let observed = RepositoryFileIdentity::read(&self.file).map_err(|source| { + SourceStructureError::Inspect { + path: self.path.clone(), + source, + } + })?; + let current = match source_root.open_file(&self.relative) { + Ok(file) => RepositoryFileIdentity::read(&file), + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(SourceStructureError::SourceFileChanged(self.path.clone())); + } + Err(OpenRepositoryFileError::Io(source)) => Err(source), + Err(OpenRepositoryFileError::NonRegular) => { + return Err(SourceStructureError::SourceFileChanged(self.path.clone())); + } + } + .map_err(|source| SourceStructureError::Inspect { + path: self.path.clone(), + source, + })?; + if observed == self.identity && current == self.identity { + Ok(()) + } else { + Err(SourceStructureError::SourceFileChanged(self.path.clone())) + } + } +} + +fn file_execution(metadata: &Metadata) -> FileExecution { + if metadata.permissions().mode() & 0o111 == 0 { + FileExecution::NonExecutable + } else { + FileExecution::Executable + } +} diff --git a/xtask/src/source_structure/tests.rs b/xtask/src/source_structure/tests.rs index a9deda9..8cf5e20 100644 --- a/xtask/src/source_structure/tests.rs +++ b/xtask/src/source_structure/tests.rs @@ -1,8 +1,9 @@ //! This module owns source-line, selection, and replacement-race tests. +mod replacement; + use super::source_kind::is_source_module; use super::{PRESENT_PATH_ARGUMENTS, SourceLineCount, exceeds_hard_limit, line_count}; -use crate::test_directory::TestDirectory; use std::io::{self, BufReader, Cursor, Read}; #[test] @@ -68,6 +69,7 @@ fn source_structure_diagnostics_are_stable() { operation: "git inventory", }); let non_regular = super::SourceStructureError::NonRegular("src/link.rs".into()); + let changed = super::SourceStructureError::SourceFileChanged("src/replaced.rs".into()); assert_eq!( framing.to_string(), "`git inventory` returned a non-NUL-terminated path" @@ -76,6 +78,10 @@ fn source_structure_diagnostics_are_stable() { non_regular.to_string(), "repository source module is not a regular file: `src/link.rs`" ); + assert_eq!( + changed.to_string(), + "repository source changed during inspection: `src/replaced.rs`" + ); let violations = super::SourceStructureError::Violations { maximum: 7, paths: vec![std::path::PathBuf::from("src/large.rs")], @@ -148,121 +154,6 @@ fn source_read_policy_enables_reads_and_refuses_blocking_io() { ); } -#[cfg(unix)] -#[test] -fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { - use std::fs; - - use crate::repository_file::RepositoryRoot; - - use super::repository_path::RepositoryPath; - use super::source_line_count; - - let directory = TestDirectory::create("source-root")?; - let root = directory.path().join("repository"); - let retained_root = directory.path().join("retained"); - fs::create_dir(&root)?; - fs::write(root.join("source.rs"), "safe\n")?; - let source_root = RepositoryRoot::open(&root)?; - let relative = RepositoryPath::admit(String::from("source.rs"))?; - - fs::rename(&root, &retained_root)?; - fs::create_dir(&root)?; - fs::write(root.join("source.rs"), "replacement\n".repeat(501))?; - - let line_count = source_line_count(&source_root, relative.as_path())?; - assert_eq!(line_count, SourceLineCount::Within(1)); - drop(source_root); - directory.close()?; - Ok(()) -} - -#[cfg(unix)] -#[test] -fn source_scan_detects_a_replaced_repository_root() -> Result<(), Box> { - use std::fs; - - use crate::repository_file::RepositoryRoot; - - let directory = TestDirectory::create("source-identity")?; - let root = directory.path().join("repository"); - let retained_root = directory.path().join("retained"); - fs::create_dir(&root)?; - let source_root = RepositoryRoot::open(&root)?; - - fs::rename(&root, &retained_root)?; - fs::create_dir(&root)?; - let result = super::verify_source_root(&source_root, &root); - assert!(matches!( - result, - Err(super::SourceStructureError::RepositoryRootChanged(ref path)) if path == &root - )); - drop(source_root); - directory.close()?; - Ok(()) -} - -#[cfg(unix)] -#[test] -fn source_open_refuses_replacement_symlink() -> Result<(), super::SourceStructureError> { - use std::fs; - use std::os::unix::fs::symlink; - - use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; - - use super::repository_path::RepositoryPath; - use super::source_line_count_with; - - let directory = TestDirectory::create("source-replacement").map_err(|source| { - super::SourceStructureError::Inspect { - path: "scoped test directory".into(), - source, - } - })?; - let root = directory.path().join("repository"); - fs::create_dir(&root).map_err(|source| super::SourceStructureError::Inspect { - path: root.clone(), - source, - })?; - let source_path = root.join("source.rs"); - let retained_path = root.join("retained.rs"); - let target_path = root.join("target.rs"); - fs::write(&source_path, "safe\n").map_err(|source| super::SourceStructureError::Inspect { - path: source_path.clone(), - source, - })?; - fs::write(&target_path, "outside\n".repeat(501)).map_err(|source| { - super::SourceStructureError::Inspect { - path: target_path.clone(), - source, - } - })?; - let source_root = - RepositoryRoot::open(&root).map_err(|source| super::SourceStructureError::Inspect { - path: root.clone(), - source, - })?; - let relative = RepositoryPath::admit(String::from("source.rs"))?; - - let result = - source_line_count_with(&source_root, relative.as_path(), |source_root, relative| { - let admitted = source_root.display_path(relative); - fs::rename(&admitted, &retained_path).map_err(OpenRepositoryFileError::Io)?; - symlink(&target_path, &admitted).map_err(OpenRepositoryFileError::Io)?; - source_root.open_file(relative) - }); - let refused = matches!( - result, - Err(super::SourceStructureError::NonRegular(ref path)) if path == &source_path - ); - assert!(refused); - drop(source_root); - directory - .close() - .map_err(|source| super::SourceStructureError::Inspect { path: root, source })?; - Ok(()) -} - #[test] fn source_module_limit_accepts_five_hundred_and_refuses_five_hundred_one() { assert!(!exceeds_hard_limit(500)); diff --git a/xtask/src/source_structure/tests/replacement.rs b/xtask/src/source_structure/tests/replacement.rs new file mode 100644 index 0000000..fd88e66 --- /dev/null +++ b/xtask/src/source_structure/tests/replacement.rs @@ -0,0 +1,137 @@ +//! This module owns source-root and source-file replacement-race tests. + +use std::fs; +use std::io; +use std::os::unix::fs::{PermissionsExt, symlink}; + +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::super::repository_path::RepositoryPath; +use super::super::source_file::{AdmittedSource, FileExecution, SourceFileAdmission}; +use super::super::{SourceLineCount, source_line_count, verify_source_root}; + +#[test] +fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { + let directory = TestDirectory::create("source-root")?; + let root = directory.path().join("repository"); + let retained_root = directory.path().join("retained"); + fs::create_dir(&root)?; + fs::write(root.join("source.rs"), "safe\n")?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + + fs::rename(&root, &retained_root)?; + fs::create_dir(&root)?; + fs::write(root.join("source.rs"), "replacement\n".repeat(501))?; + + let source = regular_source(AdmittedSource::admit(&source_root, relative.as_path())?)?; + assert_eq!( + source_line_count(&source_root, &source)?, + SourceLineCount::Within(1) + ); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_keeps_one_admitted_file_identity() -> Result<(), Box> { + let directory = TestDirectory::create("source-file-identity")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source"); + let retained_path = root.join("retained"); + fs::write(&source_path, "#!/bin/sh\nsafe\n")?; + fs::set_permissions(&source_path, fs::Permissions::from_mode(0o755))?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source"))?; + + let source = regular_source(AdmittedSource::admit(&source_root, relative.as_path())?)?; + assert_eq!(source.execution(), FileExecution::Executable); + fs::rename(&source_path, &retained_path)?; + fs::write(&source_path, "replacement\n".repeat(501))?; + + let result = source_line_count(&source_root, &source); + assert!(matches!( + result, + Err(super::super::SourceStructureError::SourceFileChanged(ref path)) + if path == &source_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_refuses_in_place_mutation() -> Result<(), Box> { + let directory = TestDirectory::create("source-file-mutation")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source.rs"); + fs::write(&source_path, "safe\n")?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + let source = regular_source(AdmittedSource::admit(&source_root, relative.as_path())?)?; + + fs::write(&source_path, "mutated\n".repeat(501))?; + + let result = source_line_count(&source_root, &source); + assert!(matches!( + result, + Err(super::super::SourceStructureError::SourceFileChanged(ref path)) + if path == &source_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_scan_detects_a_replaced_repository_root() -> Result<(), Box> { + let directory = TestDirectory::create("source-identity")?; + let root = directory.path().join("repository"); + let retained_root = directory.path().join("retained"); + fs::create_dir(&root)?; + let source_root = RepositoryRoot::open(&root)?; + + fs::rename(&root, &retained_root)?; + fs::create_dir(&root)?; + let result = verify_source_root(&source_root, &root); + assert!(matches!( + result, + Err(super::super::SourceStructureError::RepositoryRootChanged(ref path)) if path == &root + )); + drop(source_root); + directory.close()?; + Ok(()) +} + +#[test] +fn source_open_refuses_replacement_symlink() -> Result<(), Box> { + let directory = TestDirectory::create("source-replacement")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let source_path = root.join("source.rs"); + let retained_path = root.join("retained.rs"); + let target_path = root.join("target.rs"); + fs::write(&source_path, "safe\n")?; + fs::write(&target_path, "outside\n".repeat(501))?; + let source_root = RepositoryRoot::open(&root)?; + let relative = RepositoryPath::admit(String::from("source.rs"))?; + + fs::rename(&source_path, &retained_path)?; + symlink(&target_path, &source_path)?; + let admission = AdmittedSource::admit(&source_root, relative.as_path())?; + assert!(matches!(admission, SourceFileAdmission::NonRegular)); + drop(source_root); + directory.close()?; + Ok(()) +} + +fn regular_source(admission: SourceFileAdmission) -> Result { + match admission { + SourceFileAdmission::Regular(source) => Ok(source), + SourceFileAdmission::NonRegular => Err(io::Error::other("expected a regular source file")), + } +} From d15bb2a64a9de01136e6e67cf05a61b17d28f063 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:16:50 -0700 Subject: [PATCH 084/113] Fix: reserve child standard descriptors --- CHANGELOG.md | 4 +++- .../0006-descriptor-bound-child-working-directory.md | 6 +++++- xtask/src/repository_file.rs | 12 +++++++++--- xtask/src/repository_file/tests.rs | 8 ++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 xtask/src/repository_file/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 389f648..785a053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,7 +47,9 @@ after its public API and format compatibility policies are established. process authority with dedicated groups, null output, and a two-minute deadline. Documentation Git inventory and tools start from one retained repository directory handle, so transient replacement of the ambient - repository path cannot redirect validation. + repository path cannot redirect validation. Retained and per-spawn directory + descriptors are allocated at descriptor 3 or above, so child standard-stream + setup cannot overwrite the working-directory authority. Terminal signals now become typed refusals while an external repository task is active, so captured and inherited child groups are killed and reaped before `xtask` returns. Captured-output readers finish while the process-group diff --git a/docs/adr/0006-descriptor-bound-child-working-directory.md b/docs/adr/0006-descriptor-bound-child-working-directory.md index 377aa16..ddd13d5 100644 --- a/docs/adr/0006-descriptor-bound-child-working-directory.md +++ b/docs/adr/0006-descriptor-bound-child-working-directory.md @@ -33,7 +33,9 @@ between fork and exec must obey strict rules. Keep isolates the hook in the private `repository-process-spawn` workspace crate. The crate admits one operation: -1. Own a close-on-exec duplicate of the admitted repository directory. +1. Own a close-on-exec duplicate of the admitted repository directory, + allocated at descriptor 3 or above so child standard-stream setup cannot + overwrite it. 2. Register a child setup hook that calls only Rustix `fchdir`. 3. Spawn through the existing bounded process adapters. 4. Retain every selected source's device, inode, size, modification time, and @@ -44,6 +46,8 @@ crate. The crate admits one operation: POSIX specifies `fchdir` as async-signal-safe. The hook performs no allocation, locking, buffered I/O, ambient path lookup, or user callback. The descriptor closes on successful exec. A setup failure is returned by `Command::spawn`. +Both the retained process-directory handle and the per-spawn duplicate use the +same non-standard-descriptor minimum. The workspace denies unsafe code by default. Only the dedicated crate carries an explained `unsafe_code` allowance. It contains no storage, identity, format, diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs index 209d25c..61f4186 100644 --- a/xtask/src/repository_file.rs +++ b/xtask/src/repository_file.rs @@ -7,7 +7,7 @@ use std::fs::File; use std::io; -use std::os::fd::OwnedFd; +use std::os::fd::{OwnedFd, RawFd}; use std::os::unix::fs::MetadataExt as UnixMetadataExt; use std::path::{Path, PathBuf}; use std::process::{Child, Command}; @@ -38,6 +38,9 @@ pub(crate) const REPOSITORY_READ_POLICY: RepositoryReadPolicy = RepositoryReadPo blocking_io: BlockingIoPolicy::Refuse, }; +/// First descriptor outside the standard stdin, stdout, and stderr range. +const CHILD_DESCRIPTOR_MINIMUM: RawFd = 3; + pub(crate) struct RepositoryRoot { directory: Dir, identity: DirectoryIdentity, @@ -107,7 +110,7 @@ impl RepositoryRoot { /// Returns an exact child-process handle for this opened directory. pub(crate) fn process_directory(&self) -> Result { - let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, 0)?; + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, CHILD_DESCRIPTOR_MINIMUM)?; Ok(RepositoryProcessDirectory { directory }) } @@ -143,7 +146,7 @@ impl RepositoryProcessDirectory { /// The child changes directory through its retained descriptor after fork /// and before exec. Parent process state is never changed. pub(crate) fn spawn(&self, command: &mut Command) -> Result { - let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, 0)?; + let directory = rustix::io::fcntl_dupfd_cloexec(&self.directory, CHILD_DESCRIPTOR_MINIMUM)?; set_working_directory(command, directory); command.spawn() } @@ -199,3 +202,6 @@ impl From<&std::fs::Metadata> for RepositoryFileIdentity { } } } + +#[cfg(test)] +mod tests; diff --git a/xtask/src/repository_file/tests.rs b/xtask/src/repository_file/tests.rs new file mode 100644 index 0000000..da42aec --- /dev/null +++ b/xtask/src/repository_file/tests.rs @@ -0,0 +1,8 @@ +//! This module owns repository descriptor-allocation policy tests. + +use super::CHILD_DESCRIPTOR_MINIMUM; + +#[test] +fn child_directory_descriptors_stay_above_standard_streams() { + assert_eq!(CHILD_DESCRIPTOR_MINIMUM, 3); +} From dc89f7119c7e45b6feb653d01797f303836b9a66 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:22:58 -0700 Subject: [PATCH 085/113] Fix: parse combined env short options --- CHANGELOG.md | 7 +- xtask/src/source_structure/python_source.rs | 5 + .../python_source/environment.rs | 162 ++++++++---------- .../environment/short_options.rs | 30 ++++ .../python_source/environment/tests.rs | 43 +++++ 5 files changed, 155 insertions(+), 92 deletions(-) create mode 100644 xtask/src/source_structure/python_source/environment/short_options.rs create mode 100644 xtask/src/source_structure/python_source/environment/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 785a053..7f7ebb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,9 +90,10 @@ after its public API and format compatibility policies are established. `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every executable regular file regardless of filename suffix, including raw non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment - shebangs parse options, assignments, quoting, and split strings before - classifying only the selected utility, so later command arguments cannot - impersonate Python and unresolved utility substitutions fail closed. Source + shebangs parse options, combined short-option clusters, assignments, quoting, + and split strings before classifying only the selected utility, so later + command arguments cannot impersonate Python and unresolved utility + substitutions fail closed. Source execution, shebang, and physical-line evidence now come from one admitted file descriptor whose identity is revalidated after each read phase, so path replacement or in-place mutation cannot splice different file states into diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index 863a319..de8b3e5 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -135,4 +135,9 @@ mod tests { b"#!/usr/bin/env -S '${UNSET_INTERPRETER}sh'\n" )); } + + #[test] + fn combined_environment_options_cannot_hide_python() { + assert!(is_python_shebang(b"#!/usr/bin/env -S-iuFOO python3\n")); + } } diff --git a/xtask/src/source_structure/python_source/environment.rs b/xtask/src/source_structure/python_source/environment.rs index f3087c0..8145c7a 100644 --- a/xtask/src/source_structure/python_source/environment.rs +++ b/xtask/src/source_structure/python_source/environment.rs @@ -1,9 +1,11 @@ //! This module owns deterministic `env` shebang utility selection. +mod short_options; mod word_split; use std::collections::VecDeque; +use short_options::{ShortOptionAction, action as short_option_action}; use word_split::split_words; /// The selected `env` utility when repository bytes determine it safely. @@ -15,6 +17,12 @@ pub(super) enum UtilitySelection { Ambiguous, } +enum OptionAction { + Consumed, + End, + Utility, +} + /// Selects the utility executed by an `env` shebang. /// /// Invalid word framing returns absence. A selected word containing unresolved @@ -25,28 +33,13 @@ pub(super) fn selected_utility(arguments: &[u8]) -> Option { let mut split_budget = arguments.len().checked_add(1)?; while let Some(word) = words.pop_front() { if options { - if word == b"--" { - options = false; - continue; - } - if word == b"-" || is_flag(&word) { - continue; - } - if option_takes_value(&word) { - words.pop_front()?; - continue; - } - if option_has_value(&word) { - continue; - } - if let Some(split) = split_value(&word) { - split_budget = split_budget.checked_sub(1)?; - words = expanded_words(split, words)?; - options = true; - continue; - } - if word.starts_with(b"-") { - return None; + match option_action(&word, &mut words, &mut split_budget)? { + OptionAction::Consumed => continue, + OptionAction::End => { + options = false; + continue; + } + OptionAction::Utility => {} } } if word.contains(&b'=') { @@ -61,6 +54,50 @@ pub(super) fn selected_utility(arguments: &[u8]) -> Option { None } +fn option_action( + word: &[u8], + words: &mut VecDeque>, + split_budget: &mut usize, +) -> Option { + if word == b"--" { + return Some(OptionAction::End); + } + if word == b"-" || is_flag(word) { + return Some(OptionAction::Consumed); + } + if option_takes_value(word) { + words.pop_front()?; + return Some(OptionAction::Consumed); + } + if option_has_value(word) { + return Some(OptionAction::Consumed); + } + if let Some(split) = split_value(word) { + return expand_split(split, words, split_budget); + } + match short_option_action(word) { + Some(ShortOptionAction::Consumed) => Some(OptionAction::Consumed), + Some(ShortOptionAction::TakesNext) => { + words.pop_front()?; + Some(OptionAction::Consumed) + } + Some(ShortOptionAction::Split(split)) => expand_split(split, words, split_budget), + Some(ShortOptionAction::Invalid) => None, + None if word.starts_with(b"-") => None, + None => Some(OptionAction::Utility), + } +} + +fn expand_split( + split: &[u8], + words: &mut VecDeque>, + split_budget: &mut usize, +) -> Option { + *split_budget = split_budget.checked_sub(1)?; + *words = expanded_words(split, std::mem::take(words))?; + Some(OptionAction::Consumed) +} + fn expanded_words(first: &[u8], remaining: VecDeque>) -> Option>> { let mut input = first.to_vec(); for word in remaining { @@ -83,17 +120,11 @@ fn split_value(word: &[u8]) -> Option<&[u8]> { } fn option_takes_value(word: &[u8]) -> bool { - matches!( - word, - b"-u" | b"--unset" | b"-C" | b"--chdir" | b"-a" | b"--argv0" - ) + matches!(word, b"--unset" | b"--chdir" | b"--argv0") } fn option_has_value(word: &[u8]) -> bool { - [b"-u".as_slice(), b"-C", b"-a"].iter().any(|prefix| { - word.strip_prefix(*prefix) - .is_some_and(|value| !value.is_empty()) - }) || [ + [ b"--unset=".as_slice(), b"--chdir=".as_slice(), b"--argv0=".as_slice(), @@ -111,66 +142,19 @@ fn is_flag(word: &[u8]) -> bool { | b"--help" | b"--version" | b"--list-signal-handling" - ) || is_short_flag_set(word) - || [ - b"--block-signal".as_slice(), - b"--default-signal", - b"--ignore-signal", - ] - .iter() - .any(|prefix| { - word == *prefix - || word - .strip_prefix(*prefix) - .is_some_and(|value| value.starts_with(b"=")) - }) -} - -fn is_short_flag_set(word: &[u8]) -> bool { - word.strip_prefix(b"-").is_some_and(|flags| { - !flags.is_empty() && flags.iter().all(|flag| matches!(flag, b'i' | b'v' | b'0')) + ) || [ + b"--block-signal".as_slice(), + b"--default-signal", + b"--ignore-signal", + ] + .iter() + .any(|prefix| { + word == *prefix + || word + .strip_prefix(*prefix) + .is_some_and(|value| value.starts_with(b"=")) }) } #[cfg(test)] -mod tests { - #[test] - fn options_assignments_and_split_strings_precede_the_utility() { - for arguments in [ - b"-i python3 -I".as_slice(), - b"-iv python3", - b"-u PYTHONHOME python3", - b"NAME=value python3", - b"-S -i python3 -I", - b"-S \"python3 -I\"", - b"--split-string='python3 -I'", - ] { - assert_eq!( - super::selected_utility(arguments), - Some(super::UtilitySelection::Known(b"python3".to_vec())) - ); - } - } - - #[test] - fn arguments_after_the_utility_cannot_replace_it() { - for arguments in [ - b"sh -c python3".as_slice(), - b"-S sh -c 'echo python3'", - b"-S \"sh -c 'echo python3'\"", - ] { - assert_eq!( - super::selected_utility(arguments), - Some(super::UtilitySelection::Known(b"sh".to_vec())) - ); - } - } - - #[test] - fn unresolved_selected_utility_is_ambiguous() { - assert_eq!( - super::selected_utility(b"-S '${UNSET_INTERPRETER}sh'"), - Some(super::UtilitySelection::Ambiguous) - ); - } -} +mod tests; diff --git a/xtask/src/source_structure/python_source/environment/short_options.rs b/xtask/src/source_structure/python_source/environment/short_options.rs new file mode 100644 index 0000000..6fe5492 --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/short_options.rs @@ -0,0 +1,30 @@ +//! This module owns deterministic combined short-option decoding for `env`. + +pub(super) enum ShortOptionAction<'a> { + Consumed, + TakesNext, + Split(&'a [u8]), + Invalid, +} + +pub(super) fn action(word: &[u8]) -> Option> { + let mut options = word.strip_prefix(b"-")?; + if options.is_empty() || options.starts_with(b"-") { + return None; + } + loop { + let (option, remaining) = options.split_first()?; + match option { + b'i' | b'v' | b'0' if remaining.is_empty() => { + return Some(ShortOptionAction::Consumed); + } + b'i' | b'v' | b'0' => options = remaining, + b'u' | b'C' | b'a' if remaining.is_empty() => { + return Some(ShortOptionAction::TakesNext); + } + b'u' | b'C' | b'a' => return Some(ShortOptionAction::Consumed), + b'S' => return Some(ShortOptionAction::Split(remaining)), + _ => return Some(ShortOptionAction::Invalid), + } + } +} diff --git a/xtask/src/source_structure/python_source/environment/tests.rs b/xtask/src/source_structure/python_source/environment/tests.rs new file mode 100644 index 0000000..965a89e --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/tests.rs @@ -0,0 +1,43 @@ +//! This module owns `env` option and utility-selection laws. + +#[test] +fn options_assignments_and_split_strings_precede_the_utility() { + for arguments in [ + b"-i python3 -I".as_slice(), + b"-iv python3", + b"-u PYTHONHOME python3", + b"NAME=value python3", + b"-S -i python3 -I", + b"-S \"python3 -I\"", + b"-S-iuFOO python3", + b"-iSpython3 -I", + b"--split-string='python3 -I'", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"python3".to_vec())) + ); + } +} + +#[test] +fn arguments_after_the_utility_cannot_replace_it() { + for arguments in [ + b"sh -c python3".as_slice(), + b"-S sh -c 'echo python3'", + b"-S \"sh -c 'echo python3'\"", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"sh".to_vec())) + ); + } +} + +#[test] +fn unresolved_selected_utility_is_ambiguous() { + assert_eq!( + super::selected_utility(b"-S '${UNSET_INTERPRETER}sh'"), + Some(super::UtilitySelection::Ambiguous) + ); +} From c22ee2750588a394c094f970713e8178ad0eea72 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:26:25 -0700 Subject: [PATCH 086/113] Fix: sanitize documentation tool environments --- CHANGELOG.md | 4 ++- docs/dependencies/documentation-toolchain.md | 6 +++++ xtask/src/documentation_integrity/error.rs | 2 ++ .../documentation_integrity/error/display.rs | 6 +++++ .../src/documentation_integrity/execution.rs | 17 +++++++++++-- .../execution/tests.rs | 25 +++++++++++++++++-- 6 files changed, 55 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7ebb1..d03be58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,9 @@ after its public API and format compatibility policies are established. identity, refuses device, inode, size, modification-time, or change-time drift before and after each external tool, and applies a two-minute deadline across Git inventory, validation-tool - execution, and output collection. + execution, and output collection. Validation tools clear the inherited + environment and admit only the executable search path and `C` locale, so + preload hooks and host-specific configuration cannot alter evidence. Git-backed process fixtures clear the inherited environment, explicitly admit the executable search path and `C` locale, ignore system and global Git configuration, and preserve non-UTF-8 template paths without lossy diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index 7fe28ff..2c8e12f 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -73,6 +73,12 @@ the configured repository path, running checks against a substitute, and restoring the original path cannot redirect either corpus selection or validation. +Each validation tool also starts from an empty process environment. The runner +admits only the ambient executable search path needed to locate the pinned +tools and the `C` locale needed for deterministic diagnostics. Variables such +as `NODE_OPTIONS`, Git repository overrides, runtime preload hooks, and +host-specific configuration cannot enter validation-tool execution. + Each selected source also retains its device, inode, size, modification time, and change time. The Rust boundary reopens every path through the retained repository capability and compares that identity before and after each diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index ec9c96e..ff076c5 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -33,6 +33,7 @@ pub(crate) enum DocumentationError { path: String, }, EmptyCorpus(&'static str), + EnvironmentUnavailable(&'static str), GitInventory(GitInventoryError), Inspect { corpus: &'static str, @@ -158,6 +159,7 @@ impl Error for DocumentationError { | Self::CorpusTooLarge { .. } | Self::CorpusChanged { .. } | Self::EmptyCorpus(_) + | Self::EnvironmentUnavailable(_) | Self::InvalidPath { .. } | Self::NonRegular { .. } | Self::RefusalMismatch { observed: None, .. } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 9573835..49ca7aa 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -31,6 +31,12 @@ impl fmt::Display for DocumentationError { | Self::CorpusTooLarge { .. } | Self::CorpusChanged { .. } | Self::EmptyCorpus(_)) => corpus(formatter, error), + Self::EnvironmentUnavailable(variable) => { + write!( + formatter, + "documentation tool environment variable `{variable}` is unavailable" + ) + } Self::GitInventory(error) => write!(formatter, "{error}"), Self::Inspect { corpus, path, .. } => { source_path(formatter, SourcePathDiagnostic::Inspect, corpus, path) diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index 2f2bba1..c3ca313 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -3,6 +3,8 @@ mod corpus_guard; mod refusal_check; +use std::env; +use std::ffi::OsStr; use std::process::{Command, Stdio}; use std::time::Duration; @@ -152,8 +154,8 @@ impl ToolRunner for ExternalToolRunner<'_> { tool: DocumentationTool, arguments: &[String], ) -> Result { - let mut command = Command::new(tool.program()); - command.args(arguments).stdin(Stdio::null()); + let path = env::var_os("PATH").ok_or(DocumentationError::EnvironmentUnavailable("PATH"))?; + let mut command = documentation_command(tool, arguments, &path); bounded_process::capture_with( tool.program(), &mut command, @@ -174,6 +176,17 @@ impl ToolRunner for ExternalToolRunner<'_> { } } +fn documentation_command(tool: DocumentationTool, arguments: &[String], path: &OsStr) -> Command { + let mut command = Command::new(tool.program()); + command + .args(arguments) + .stdin(Stdio::null()) + .env_clear() + .env("PATH", path) + .env("LC_ALL", "C"); + command +} + #[cfg(test)] #[path = "execution/tests.rs"] mod tests; diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index 2805a45..62550bd 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -1,4 +1,5 @@ -use std::collections::VecDeque; +use std::collections::{BTreeMap, VecDeque}; +use std::ffi::OsString; use std::fs; use std::path::PathBuf; @@ -8,7 +9,7 @@ use crate::repository_file::RepositoryRoot; use crate::test_directory::TestDirectory; use super::corpus_guard::CorpusGuardedRunner; -use super::{DocumentationError, DocumentationTool, ToolRunner}; +use super::{DocumentationError, DocumentationTool, ToolRunner, documentation_command}; struct RecordingRunner { calls: Vec<(DocumentationTool, Vec)>, @@ -20,6 +21,26 @@ struct ReplacingRunner { retained: PathBuf, } +#[test] +fn documentation_tools_receive_only_reviewed_environment() { + let path = OsString::from("/reviewed/tools"); + let command = documentation_command(DocumentationTool::Markdownlint, &[], &path); + let observed = command + .get_envs() + .map(|(name, value)| (name.to_owned(), value.map(OsString::from))) + .collect::>(); + let expected = BTreeMap::from([ + (OsString::from("LC_ALL"), Some(OsString::from("C"))), + (OsString::from("PATH"), Some(path)), + ]); + + assert_eq!(observed, expected); + assert_eq!( + DocumentationError::EnvironmentUnavailable("PATH").to_string(), + "documentation tool environment variable `PATH` is unavailable" + ); +} + #[test] fn admitted_tools_run_with_exact_arguments_and_silent_success() { let mut runner = RecordingRunner::new([ From 082664165353dc3db9adb52165026a399e3e8b72 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:28:24 -0700 Subject: [PATCH 087/113] Fix: sanitize Git inventory environments --- CHANGELOG.md | 3 ++ docs/dependencies/documentation-toolchain.md | 7 ++++ xtask/src/git_inventory/process.rs | 25 +++++++++++- xtask/src/git_inventory/process/tests.rs | 40 +++++++++++++++++++- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d03be58..bd34d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,9 @@ after its public API and format compatibility policies are established. execution, and output collection. Validation tools clear the inherited environment and admit only the executable search path and `C` locale, so preload hooks and host-specific configuration cannot alter evidence. + Git inventory uses a separate explicit profile that also nulls system and + global configuration and disables optional locking, so repository overrides + cannot redirect selection or cause incidental index writes. Git-backed process fixtures clear the inherited environment, explicitly admit the executable search path and `C` locale, ignore system and global Git configuration, and preserve non-UTF-8 template paths without lossy diff --git a/docs/dependencies/documentation-toolchain.md b/docs/dependencies/documentation-toolchain.md index 2c8e12f..a7ef879 100644 --- a/docs/dependencies/documentation-toolchain.md +++ b/docs/dependencies/documentation-toolchain.md @@ -57,6 +57,13 @@ at most the 16 MiB path-stream bound, and diagnostics retain at most 64 KiB. A timeout, terminal signal, reader failure, or exceeded bound terminates the whole group and reaps the child before the task refuses. +Inventory commands also clear the inherited environment. They admit only the +executable search path, the `C` locale, null system and global Git +configuration, and disabled optional locking. Repository directory, worktree, +index, object, and configuration overrides therefore cannot redirect the +selected path set, and read-only inventory cannot opportunistically refresh +the index. + Documentation corpus tests construct their Git fixtures through the same bounded process authority. Each fixture command runs in a dedicated process group with a two-minute deadline and null output streams, so a stalled command diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index c9687b8..9115cde 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -1,6 +1,8 @@ //! This module owns deadline-bounded Git path process execution. use std::collections::BTreeSet; +use std::env; +use std::ffi::OsStr; use std::io; use std::path::Path; use std::process::{Child, Command, Stdio}; @@ -48,8 +50,12 @@ fn paths_with_deadline( deadline: Duration, spawn: impl FnOnce(&mut Command) -> Result, ) -> Result, GitInventoryError> { - let mut command = Command::new("git"); - command.args(arguments).stdin(Stdio::null()); + let path = env::var_os("PATH").ok_or_else(|| GitInventoryError::Run { + operation, + action: "read PATH for", + source: io::Error::new(io::ErrorKind::NotFound, "PATH is unavailable"), + })?; + let mut command = git_command(arguments, &path); let output = bounded_process::capture_with_limits( "git", &mut command, @@ -66,6 +72,21 @@ fn paths_with_deadline( } } +fn git_command(arguments: &[&str], path: &OsStr) -> Command { + let mut command = Command::new("git"); + command + .args(arguments) + .stdin(Stdio::null()) + .env_clear() + .env("PATH", path) + .env("LC_ALL", "C") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .env("GIT_OPTIONAL_LOCKS", "0"); + command +} + fn process_failure(operation: &'static str, source: ProcessError) -> GitInventoryError { match source { ProcessError::OutputLimit { diff --git a/xtask/src/git_inventory/process/tests.rs b/xtask/src/git_inventory/process/tests.rs index 680129c..8ad5136 100644 --- a/xtask/src/git_inventory/process/tests.rs +++ b/xtask/src/git_inventory/process/tests.rs @@ -1,18 +1,56 @@ //! This module owns Git process deadline and diagnostic regression evidence. +use std::collections::BTreeMap; use std::env; +use std::ffi::OsString; use std::process::{Command, Stdio}; use std::time::Duration; use std::os::unix::process::CommandExt; -use super::{GIT_DIAGNOSTIC_LIMIT_BYTES, git_failure, paths_with_deadline, process_failure}; +use super::{ + GIT_DIAGNOSTIC_LIMIT_BYTES, git_command, git_failure, paths_with_deadline, process_failure, +}; use crate::bounded_process::ProcessError; use crate::git_inventory::GitInventoryError; const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_GIT_CHILD"; const PARKED_CHILD_TEST: &str = "git_inventory::process::tests::process_child_parks_indefinitely"; +#[test] +fn git_inventory_receives_only_reviewed_environment() { + let command = git_command(&["status"], OsString::from("/reviewed/tools").as_os_str()); + let observed = command + .get_envs() + .map(|(name, value)| (name.to_owned(), value.map(OsString::from))) + .collect::>(); + let expected = BTreeMap::from([ + ( + OsString::from("GIT_CONFIG_COUNT"), + Some(OsString::from("0")), + ), + ( + OsString::from("GIT_CONFIG_GLOBAL"), + Some(OsString::from("/dev/null")), + ), + ( + OsString::from("GIT_CONFIG_NOSYSTEM"), + Some(OsString::from("1")), + ), + ( + OsString::from("GIT_OPTIONAL_LOCKS"), + Some(OsString::from("0")), + ), + (OsString::from("LC_ALL"), Some(OsString::from("C"))), + ( + OsString::from("PATH"), + Some(OsString::from("/reviewed/tools")), + ), + ]); + + assert_eq!(observed, expected); +} + #[test] fn git_diagnostic_encoding_failure_retains_exit_status() { let error = git_failure("test diagnostics", Some(9), vec![u8::MAX]); From 8a4085828414a64d545d7d4653d3aeb380ad13e0 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:35:26 -0700 Subject: [PATCH 088/113] Fix: bind source inventory to repository authority --- CHANGELOG.md | 9 ++++++--- docs/Rust Standards.md | 5 +++++ ...escriptor-bound-child-working-directory.md | 11 ++++++----- xtask/src/git_inventory.rs | 2 +- xtask/src/git_inventory/process.rs | 15 --------------- xtask/src/source_structure.rs | 10 +++++++++- .../src/source_structure/source_inventory.rs | 19 ++++++++++--------- xtask/tests/source_policy_contract.rs | 13 +++++++++++-- 8 files changed, 48 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd34d6e..cc9080f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,9 +89,12 @@ after its public API and format compatibility policies are established. executable sources regardless of filename suffix. - Repository source verification now uses capability-relative, no-follow file opens, normalizes symlink refusal at that capability boundary across Unix - error conventions, and verifies repository-root identity after Git inventory - and again after source scanning, so a persistent root replacement or source - path replaced with a symlink is refused. The pure Rust boundary also refuses + error conventions, starts Git inventory through a descriptor duplicated from + that same admitted root, and verifies repository-root identity before + inventory, after inventory, and after source scanning. Persistent or + transient ambient-root substitution therefore cannot split path selection + from source reads, and a source path replaced with a symlink is refused. The + pure Rust boundary also refuses `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every executable regular file regardless of filename suffix, including raw non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment diff --git a/docs/Rust Standards.md b/docs/Rust Standards.md index 3be3ada..0905782 100644 --- a/docs/Rust Standards.md +++ b/docs/Rust Standards.md @@ -485,6 +485,11 @@ different files. The scanner MUST revalidate both the descriptor identity and the current repository path after each read phase, and refuse concurrent replacement or in-place mutation. +Source Git inventory and source-file admission MUST descend from the same +opened repository capability. Inventory MUST start through a descriptor-bound +child working directory; an ambient pathname passed to +`Command::current_dir` cannot establish the required authority continuity. + A file above 300 lines MUST begin with a decomposition issue or contain an approved exception explaining why splitting it would damage locality. A file above 500 lines does not merge. diff --git a/docs/adr/0006-descriptor-bound-child-working-directory.md b/docs/adr/0006-descriptor-bound-child-working-directory.md index ddd13d5..429761a 100644 --- a/docs/adr/0006-descriptor-bound-child-working-directory.md +++ b/docs/adr/0006-descriptor-bound-child-working-directory.md @@ -8,11 +8,12 @@ ## Context -Documentation verification inventories Git paths and runs pinned validation +Repository verification inventories Git paths and runs pinned validation tools. Opening the repository as a capability protects file reads, but passing its ambient pathname to a child creates another replacement window. An attacker could move the admitted directory, substitute another repository -while the tools run, and restore the original before the final identity check. +while source or documentation inventory runs, and restore the original before +the final identity check. Retaining the directory alone does not bind a selected corpus path to the file admitted at that path. A source can be renamed, replaced while a tool reads it, @@ -71,9 +72,9 @@ network, or application policy. ## Consequences -Git inventory and documentation tools start in the exact opened repository -even if its pathname is replaced. Parent process state remains unchanged, so -parallel tests and readers are deterministic. +Source inventory, documentation inventory, and documentation tools start in +the exact opened repository even if its pathname is replaced. Parent process +state remains unchanged, so parallel tests and readers are deterministic. External tool output is admitted only while every selected path still has the admitted device, inode, size, modification time, and change time diff --git a/xtask/src/git_inventory.rs b/xtask/src/git_inventory.rs index 917d6f6..e1df1d5 100644 --- a/xtask/src/git_inventory.rs +++ b/xtask/src/git_inventory.rs @@ -6,4 +6,4 @@ mod process; pub(crate) use error::{GitInventoryError, GitOutputUnit}; pub(crate) use path_stream::GitPath; -pub(crate) use process::{paths, paths_with}; +pub(crate) use process::paths_with; diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index 9115cde..f4b0b3a 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -4,7 +4,6 @@ use std::collections::BTreeSet; use std::env; use std::ffi::OsStr; use std::io; -use std::path::Path; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -18,20 +17,6 @@ const GIT_DIAGNOSTIC_LIMIT_BYTES: usize = 65_536; const GIT_CAPTURE_LIMITS: CaptureLimits = CaptureLimits::new(GIT_PATH_STREAM_LIMIT_BYTES, GIT_DIAGNOSTIC_LIMIT_BYTES); -/// Lists repository paths through a deadline-bounded Git process group. -/// -/// The adapter materializes at most the 16 MiB path-stream bound before -/// deterministic NUL-framed decoding. -pub(crate) fn paths( - repository_root: &Path, - arguments: &[&str], - operation: &'static str, -) -> Result, GitInventoryError> { - paths_with(arguments, operation, |command| { - command.current_dir(repository_root).spawn() - }) -} - /// Lists paths through an injected capability-bound spawn operation. /// /// The adapter materializes at most the 16 MiB path-stream bound before diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index c11fc6b..aee74ab 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -29,7 +29,15 @@ pub(super) fn check(repository_root: &Path) -> Result<(), SourceStructureError> path: repository_root.to_owned(), source, })?; - let paths = source_paths(repository_root)?; + verify_source_root(&source_root, repository_root)?; + let process_directory = + source_root + .process_directory() + .map_err(|source| SourceStructureError::Inspect { + path: repository_root.to_owned(), + source, + })?; + let paths = source_paths(&process_directory)?; verify_source_root(&source_root, repository_root)?; let violations = inventory_violations(&source_root, paths)?; verify_source_root(&source_root, repository_root)?; diff --git a/xtask/src/source_structure/source_inventory.rs b/xtask/src/source_structure/source_inventory.rs index d20bf5a..90d2cba 100644 --- a/xtask/src/source_structure/source_inventory.rs +++ b/xtask/src/source_structure/source_inventory.rs @@ -5,7 +5,8 @@ use std::ffi::OsString; use std::os::unix::ffi::OsStringExt; use std::path::{Component, Path, PathBuf}; -use crate::git_inventory::{GitPath, paths as git_paths}; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::RepositoryProcessDirectory; use super::repository_path::RepositoryPath; use super::source_error::SourceStructureError; @@ -34,16 +35,16 @@ impl InspectionPath { } } -pub(super) fn collect(repository_root: &Path) -> Result { - let present = git_paths( - repository_root, - &PRESENT_PATH_ARGUMENTS, - "git ls-files present", - )?; - let deleted = git_paths( - repository_root, +pub(super) fn collect( + process_directory: &RepositoryProcessDirectory, +) -> Result { + let present = paths_with(&PRESENT_PATH_ARGUMENTS, "git ls-files present", |command| { + process_directory.spawn(command) + })?; + let deleted = paths_with( &["ls-files", "-z", "--deleted"], "git ls-files deleted", + |command| process_directory.spawn(command), )?; select(&present, &deleted) } diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 5eae83b..092cc53 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -18,6 +18,7 @@ const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs" const GIT_PROCESS: &str = include_str!("../src/git_inventory/process.rs"); const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); +const SOURCE_INVENTORY: &str = include_str!("../src/source_structure/source_inventory.rs"); #[test] fn written_source_limit_matches_the_executable_law() { @@ -42,10 +43,18 @@ fn source_scan_revalidates_repository_identity_after_reading() { SOURCE_STRUCTURE .matches("verify_source_root(&source_root, repository_root)?;") .count(), - 2 + 3 ); } +#[test] +fn source_inventory_uses_the_admitted_repository_directory() { + assert!(SOURCE_STRUCTURE.contains(".process_directory()")); + assert!(SOURCE_INVENTORY.contains("paths_with(")); + assert!(!SOURCE_INVENTORY.contains("paths as git_paths")); + assert!(!GIT_PROCESS.contains("current_dir(")); +} + #[test] fn git_inventory_uses_the_deadline_bounded_process_layer() { assert!(GIT_PROCESS.contains("const GIT_DEADLINE: Duration")); @@ -220,7 +229,7 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " pub(crate) fn as_bytes(", ], )?; - require_docs(GIT_PROCESS, &["pub(crate) fn paths("]) + require_docs(GIT_PROCESS, &["pub(crate) fn paths_with("]) } fn require_docs(source: &str, declarations: &[&str]) -> Result<(), String> { From 5f04483d79901df5a9f194a01eeb9e3c580aff8c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:41:00 -0700 Subject: [PATCH 089/113] Fix: split documentation refusal diagnostics --- .../documentation_integrity/error/display.rs | 20 +++++--------- .../error/display/refusal.rs | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 xtask/src/documentation_integrity/error/display/refusal.rs diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 49ca7aa..ca1ce6c 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -6,6 +6,10 @@ use crate::diagnostic::{escaped_controls, escaped_path}; use super::DocumentationError; +mod refusal; + +use refusal::{refusal_fixture, refusal_mismatch}; + #[derive(Clone, Copy)] enum SourcePathDiagnostic { Changed, @@ -51,21 +55,9 @@ impl fmt::Display for DocumentationError { write!(formatter, "{corpus} corpus contains a non-UTF-8 path") } Self::Process(error) => write!(formatter, "{error}"), - Self::RefusalFixture { action, .. } => { - write!( - formatter, - "cannot {action} for documentation refusal evidence" - ) - } + Self::RefusalFixture { action, .. } => refusal_fixture(formatter, action), Self::RefusalMismatch { scenario, observed } => { - write!( - formatter, - "documentation refusal scenario `{scenario}` did not produce its reviewed error" - )?; - if let Some(error) = observed { - write!(formatter, ": {error}")?; - } - Ok(()) + refusal_mismatch(formatter, scenario, observed.as_deref()) } error @ (Self::RepositoryFileEncoding { .. } | Self::RepositoryFileInspect { .. } diff --git a/xtask/src/documentation_integrity/error/display/refusal.rs b/xtask/src/documentation_integrity/error/display/refusal.rs new file mode 100644 index 0000000..2be3ba6 --- /dev/null +++ b/xtask/src/documentation_integrity/error/display/refusal.rs @@ -0,0 +1,27 @@ +//! This module owns human-readable documentation-refusal diagnostics. + +use std::fmt; + +use super::super::DocumentationError; + +pub(super) fn refusal_fixture(formatter: &mut fmt::Formatter<'_>, action: &str) -> fmt::Result { + write!( + formatter, + "cannot {action} for documentation refusal evidence" + ) +} + +pub(super) fn refusal_mismatch( + formatter: &mut fmt::Formatter<'_>, + scenario: &str, + observed: Option<&DocumentationError>, +) -> fmt::Result { + write!( + formatter, + "documentation refusal scenario `{scenario}` did not produce its reviewed error" + )?; + if let Some(error) = observed { + write!(formatter, ": {error}")?; + } + Ok(()) +} From b16e4a4ca0f0064ceb3b5136ede33e3aab76350a Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:42:03 -0700 Subject: [PATCH 090/113] Fix: hermetically mark documentation tool execution --- xtask/tests/cli_contract/documentation_tools.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/xtask/tests/cli_contract/documentation_tools.rs b/xtask/tests/cli_contract/documentation_tools.rs index cb86238..7cd8695 100644 --- a/xtask/tests/cli_contract/documentation_tools.rs +++ b/xtask/tests/cli_contract/documentation_tools.rs @@ -15,7 +15,6 @@ use std::process::{Command, Output}; use std::sync::atomic::{AtomicU64, Ordering}; static SEQUENCE: AtomicU64 = AtomicU64::new(0); -const MARKER_ENVIRONMENT: &str = "KEEP_TEST_TOOL_MARKERS"; pub(crate) struct DocumentationTools { root: Option, @@ -48,7 +47,6 @@ impl DocumentationTools { Command::new(env!("CARGO_BIN_EXE_xtask")) .args(arguments) .env("PATH", self.path_environment()?) - .env(MARKER_ENVIRONMENT, self.markers()?) .output() } @@ -78,9 +76,10 @@ impl DocumentationTools { version_argument: &str, version: &str, ) -> Result<(), io::Error> { + let marker = shell_word(&self.markers()?.join(program))?; let script = format!( "#!/bin/sh\n\ - : > \"${{{MARKER_ENVIRONMENT}}}/{program}\"\n\ + : > {marker}\n\ for argument in \"$@\"; do\n\ \x20 if [ \"$argument\" = \"{version_argument}\" ]; then\n\ \x20 printf '%s\\n' '{version}'\n\ @@ -117,6 +116,13 @@ impl DocumentationTools { } } +fn shell_word(path: &Path) -> Result { + let text = path + .to_str() + .ok_or_else(|| io::Error::other("documentation tool path is not UTF-8"))?; + Ok(format!("'{}'", text.replace('\'', "'\\''"))) +} + impl Drop for DocumentationTools { fn drop(&mut self) { if let Some(root) = self.root.take() { From 936dda5a63177ed63581172a6c8ddbf62e0fbe64 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 19:46:02 -0700 Subject: [PATCH 091/113] Fix: admit reviewed fuzz dependency licenses --- CHANGELOG.md | 3 +++ docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md | 6 ++++++ fuzz/deny.toml | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9080f..31859b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,9 @@ after its public API and format compatibility policies are established. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Run deadlines use checked addition of the exploration budget and process-grace interval before process-group execution. +- The fuzz dependency-policy gate now grants exact MIT license exceptions to + the reviewed `memchr` 2.8.3 and `zmij` 1.0.23 transitive dependencies while + retaining Apache-2.0 as the default license allowlist. - ChunkId v1 and CDC profile v1 conformance now run through one bounded Rust `cargo xtask conformance-check` command, including the external `b3sum` witness, reproducible Gear-table recipe, scalar and streaming FastCDC laws, diff --git a/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md index a019a7a..d5286d0 100644 --- a/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md +++ b/docs/dependencies/serde-and-serde-json-1.0.229-1.0.151.md @@ -53,6 +53,12 @@ workspace lockfile. Their manifests declare minimum supported Rust versions below Keep's pinned toolchain. +The fuzz-workspace license policy retains Apache-2.0 as its default allowlist +and grants exact-version MIT exceptions to `memchr` 2.8.3 and `zmij` 1.0.23. +Those exceptions admit only the reviewed transitive graph named above; a +resolved version change remains a policy failure until this record and the +exception are reviewed together. + Keep-owned code invokes only safe APIs. The parser and its transitive dependencies may contain implementation details outside Keep's `unsafe_code` lint boundary, so `cargo deny` and RustSec checks remain mandatory diff --git a/fuzz/deny.toml b/fuzz/deny.toml index 9e12454..5eae9d4 100644 --- a/fuzz/deny.toml +++ b/fuzz/deny.toml @@ -16,6 +16,10 @@ exceptions = [ { allow = ["BSD-2-Clause"], crate = "arrayref@0.3.9" }, # `libfuzzer-sys` is confined to this non-published fuzz workspace. { allow = ["MIT", "NCSA"], crate = "libfuzzer-sys@0.4.13" }, + # `memchr` is a reviewed transitive dependency of locked serde_json 1.0.151. + { allow = ["MIT"], crate = "memchr@2.8.3" }, + # `zmij` is a reviewed transitive dependency of locked serde_json 1.0.151. + { allow = ["MIT"], crate = "zmij@1.0.23" }, ] [sources] From 28fb59c5cb92c8c07b877e74ceedc8790e9b6674 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:04:27 -0700 Subject: [PATCH 092/113] Fix: preserve Git failure precedence --- CHANGELOG.md | 5 ++++- xtask/src/git_inventory/process.rs | 17 +++++++++------ xtask/src/git_inventory/process/tests.rs | 27 ++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31859b0..c650b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,10 @@ after its public API and format compatibility policies are established. preload hooks and host-specific configuration cannot alter evidence. Git inventory uses a separate explicit profile that also nulls system and global configuration and disables optional locking, so repository overrides - cannot redirect selection or cause incidental index writes. + cannot redirect selection or cause incidental index writes. Failed Git + inventory commands report their exit status and diagnostic before attempting + path-stream decoding, so malformed stdout cannot mask the authoritative + failure. Git-backed process fixtures clear the inherited environment, explicitly admit the executable search path and `C` locale, ignore system and global Git configuration, and preserve non-UTF-8 template paths without lossy diff --git a/xtask/src/git_inventory/process.rs b/xtask/src/git_inventory/process.rs index f4b0b3a..9ce7036 100644 --- a/xtask/src/git_inventory/process.rs +++ b/xtask/src/git_inventory/process.rs @@ -7,7 +7,7 @@ use std::io; use std::process::{Child, Command, Stdio}; use std::time::Duration; -use crate::bounded_process::{self, CaptureLimits, ProcessError}; +use crate::bounded_process::{self, CaptureLimits, ProcessError, ProcessOutput}; use super::path_stream::{GIT_PATH_STREAM_LIMIT_BYTES, GitPath, read_paths}; use super::{GitInventoryError, GitOutputUnit}; @@ -49,12 +49,17 @@ fn paths_with_deadline( spawn, ) .map_err(|source| process_failure(operation, source))?; - let paths = read_paths(output.stdout.as_slice(), operation)?; - if output.succeeded { - Ok(paths) - } else { - Err(git_failure(operation, output.code, output.stderr)) + inventory_output(operation, output) +} + +fn inventory_output( + operation: &'static str, + output: ProcessOutput, +) -> Result, GitInventoryError> { + if !output.succeeded { + return Err(git_failure(operation, output.code, output.stderr)); } + read_paths(output.stdout.as_slice(), operation) } fn git_command(arguments: &[&str], path: &OsStr) -> Command { diff --git a/xtask/src/git_inventory/process/tests.rs b/xtask/src/git_inventory/process/tests.rs index 8ad5136..d9b3c89 100644 --- a/xtask/src/git_inventory/process/tests.rs +++ b/xtask/src/git_inventory/process/tests.rs @@ -9,9 +9,10 @@ use std::time::Duration; use std::os::unix::process::CommandExt; use super::{ - GIT_DIAGNOSTIC_LIMIT_BYTES, git_command, git_failure, paths_with_deadline, process_failure, + GIT_DIAGNOSTIC_LIMIT_BYTES, git_command, git_failure, inventory_output, paths_with_deadline, + process_failure, }; -use crate::bounded_process::ProcessError; +use crate::bounded_process::{ProcessError, ProcessOutput}; use crate::git_inventory::GitInventoryError; const PARKED_CHILD: &str = "KEEP_XTASK_PARKED_GIT_CHILD"; @@ -64,6 +65,28 @@ fn git_diagnostic_encoding_failure_retains_exit_status() { )); } +#[test] +fn failed_git_status_precedes_stdout_decoding() { + let result = inventory_output( + "test failure precedence", + ProcessOutput { + code: Some(23), + succeeded: false, + stdout: b"unterminated".to_vec(), + stderr: b"fatal: reviewed failure\n".to_vec(), + }, + ); + + assert!(matches!( + result, + Err(GitInventoryError::Failed { + operation: "test failure precedence", + code: Some(23), + ref stderr, + }) if stderr == "fatal: reviewed failure\n" + )); +} + #[test] fn git_diagnostic_limit_maps_to_the_inventory_boundary() { let error = process_failure( From d20c6baacf4b792fe7c5864a66e213deb08a734e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:05:47 -0700 Subject: [PATCH 093/113] Fix: reject unclassified workflow steps --- CHANGELOG.md | 7 ++++--- .../workflow_contract.rs | 2 ++ .../workflow_contract/tests.rs | 21 +++++++++++++++---- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c650b3a..db1d52f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,9 +28,10 @@ after its public API and format compatibility policies are established. unreviewed workflow/job run defaults and step execution fields, pins the documentation runner and job deadline, requires the exact top-level `contents: read` permission mapping, rejects guarded or failure-tolerant - documentation jobs and required steps, requires each Dependabot update block - to choose exactly one directory field form, and requires the documentation - workflow to run for pushes to `main` and every pull request, + documentation jobs and required steps, refuses step mappings that define + neither a reviewed action nor a run command, requires each Dependabot update + block to choose exactly one directory field form, and requires the + documentation workflow to run for pushes to `main` and every pull request, executes malformed Markdown and workflow evidence through the named `cargo xtask documentation-refusal-check` boundary instead of a zero-match-successful libtest substring filter, diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 17fae2d..dbe3897 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -123,6 +123,8 @@ fn reviewed_steps(steps: &[Yaml]) -> Result, Documentatio }); } else if let Some(run) = admit_run(step)? { admitted.push(run); + } else { + return Err(contract("documentation job steps are reviewed")); } } if actions diff --git a/xtask/src/documentation_integrity/workflow_contract/tests.rs b/xtask/src/documentation_integrity/workflow_contract/tests.rs index ca51fa4..b4e3629 100644 --- a/xtask/src/documentation_integrity/workflow_contract/tests.rs +++ b/xtask/src/documentation_integrity/workflow_contract/tests.rs @@ -86,6 +86,22 @@ fn documentation_job_requires_the_malformed_input_regressions() { assert!(!super::steps_have_reviewed_membership(&steps)); } +#[test] +fn documentation_job_refuses_unclassified_step_mappings() { + let workflow = WORKFLOW.replace( + " - name: Verify malformed inputs\n", + " - name: Unreviewed placeholder\n - name: Verify malformed inputs\n", + ); + + assert!(matches!( + super::admit(&workflow), + Err(super::DocumentationError::RepositoryContract { + path: super::CI_PATH, + requirement: "documentation job steps are reviewed", + }) + )); +} + #[test] fn documentation_job_refuses_python_execution() { let workflow = WORKFLOW.replace( @@ -134,10 +150,7 @@ jobs: super::admit(workflow), Err(super::DocumentationError::RepositoryContract { path: super::CI_PATH, - requirement: concat!( - "documentation job run commands are reviewed and required ", - "commands execute once" - ), + requirement: "documentation job steps are reviewed", }) )); } From 222621a17add92f9604666adae5384e30a5ffd2d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:08:17 -0700 Subject: [PATCH 094/113] Fix: refuse nonregular executable candidates --- CHANGELOG.md | 2 ++ xtask/src/source_structure.rs | 4 ++- .../src/source_structure/tests/replacement.rs | 30 ++++++++++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db1d52f..593dba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ after its public API and format compatibility policies are established. repository path cannot redirect validation. Retained and per-spawn directory descriptors are allocated at descriptor 3 or above, so child standard-stream setup cannot overwrite the working-directory authority. + Source-structure inspection refuses nonregular executable candidates instead + of silently omitting them from the Python and hard-line-limit policy. Terminal signals now become typed refusals while an external repository task is active, so captured and inherited child groups are killed and reaped before `xtask` returns. Captured-output readers finish while the process-group diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index aee74ab..2912917 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -76,7 +76,9 @@ fn inventory_violations( let SourceFileAdmission::Regular(source) = AdmittedSource::admit(source_root, relative.as_path())? else { - continue; + return Err(SourceStructureError::NonRegular( + source_root.display_path(relative.as_path()), + )); }; if source.execution() == FileExecution::Executable && source_line_count(source_root, &source)? == SourceLineCount::Exceeded diff --git a/xtask/src/source_structure/tests/replacement.rs b/xtask/src/source_structure/tests/replacement.rs index fd88e66..40f6d39 100644 --- a/xtask/src/source_structure/tests/replacement.rs +++ b/xtask/src/source_structure/tests/replacement.rs @@ -1,5 +1,6 @@ //! This module owns source-root and source-file replacement-race tests. +use std::collections::BTreeSet; use std::fs; use std::io; use std::os::unix::fs::{PermissionsExt, symlink}; @@ -9,7 +10,9 @@ use crate::test_directory::TestDirectory; use super::super::repository_path::RepositoryPath; use super::super::source_file::{AdmittedSource, FileExecution, SourceFileAdmission}; -use super::super::{SourceLineCount, source_line_count, verify_source_root}; +use super::super::source_inventory; +use super::super::{SourceLineCount, inventory_violations, source_line_count, verify_source_root}; +use crate::git_inventory::GitPath; #[test] fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box> { @@ -129,6 +132,31 @@ fn source_open_refuses_replacement_symlink() -> Result<(), Box Result<(), Box> { + let directory = TestDirectory::create("nonregular-executable-candidate")?; + let root = directory.path().join("repository"); + fs::create_dir(&root)?; + let candidate_path = root.join("script.bin"); + let target_path = root.join("target"); + fs::write(&target_path, "#!/bin/sh\n")?; + symlink(&target_path, &candidate_path)?; + let source_root = RepositoryRoot::open(&root)?; + let present = BTreeSet::from([GitPath::new(b"script.bin".to_vec())]); + let inventory = source_inventory::select(&present, &BTreeSet::new())?; + + let result = inventory_violations(&source_root, inventory); + + assert!(matches!( + result, + Err(super::super::SourceStructureError::NonRegular(ref path)) + if path == &candidate_path + )); + drop(source_root); + directory.close()?; + Ok(()) +} + fn regular_source(admission: SourceFileAdmission) -> Result { match admission { SourceFileAdmission::Regular(source) => Ok(source), From c3bf334d00139bc4507ff2b38dcb7eace76c858f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:11:47 -0700 Subject: [PATCH 095/113] Fix: parse abbreviated env long options --- CHANGELOG.md | 8 +- xtask/src/source_structure/python_source.rs | 2 + .../python_source/environment.rs | 67 +++--------- .../python_source/environment/long_options.rs | 100 ++++++++++++++++++ .../python_source/environment/tests.rs | 14 +++ 5 files changed, 137 insertions(+), 54 deletions(-) create mode 100644 xtask/src/source_structure/python_source/environment/long_options.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 593dba6..4f111d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,10 +107,10 @@ after its public API and format compatibility policies are established. `.py`, `.pyw`, dot-only Python basenames, and Python shebangs in every executable regular file regardless of filename suffix, including raw non-UTF-8 Git paths and attached `env -S` interpreter strings. Environment - shebangs parse options, combined short-option clusters, assignments, quoting, - and split strings before classifying only the selected utility, so later - command arguments cannot impersonate Python and unresolved utility - substitutions fail closed. Source + shebangs parse exact and unambiguous abbreviated long options, combined + short-option clusters, assignments, quoting, and split strings before + classifying only the selected utility, so later command arguments cannot + impersonate Python and unresolved utility substitutions fail closed. Source execution, shebang, and physical-line evidence now come from one admitted file descriptor whose identity is revalidated after each read phase, so path replacement or in-place mutation cannot splice different file states into diff --git a/xtask/src/source_structure/python_source.rs b/xtask/src/source_structure/python_source.rs index de8b3e5..01d470b 100644 --- a/xtask/src/source_structure/python_source.rs +++ b/xtask/src/source_structure/python_source.rs @@ -108,6 +108,8 @@ mod tests { b"#!/usr/bin/env -Spython3 -I\n", b"#!/usr/bin/env -S/opt/PyPy3 -I\n", b"#!/usr/bin/env --split-string=python3\n", + b"#!/usr/bin/env --spl=python3 -I\n", + b"#!/usr/bin/env --ignore-e python3\n", b"#!/opt/PyPy3\n", ] { assert!(is_python_shebang(shebang)); diff --git a/xtask/src/source_structure/python_source/environment.rs b/xtask/src/source_structure/python_source/environment.rs index 8145c7a..e3a2443 100644 --- a/xtask/src/source_structure/python_source/environment.rs +++ b/xtask/src/source_structure/python_source/environment.rs @@ -1,10 +1,12 @@ //! This module owns deterministic `env` shebang utility selection. +mod long_options; mod short_options; mod word_split; use std::collections::VecDeque; +use long_options::{LongOptionAction, action as long_option_action}; use short_options::{ShortOptionAction, action as short_option_action}; use word_split::split_words; @@ -62,17 +64,21 @@ fn option_action( if word == b"--" { return Some(OptionAction::End); } - if word == b"-" || is_flag(word) { + if word == b"-" { return Some(OptionAction::Consumed); } - if option_takes_value(word) { - words.pop_front()?; - return Some(OptionAction::Consumed); - } - if option_has_value(word) { - return Some(OptionAction::Consumed); + if let Some(action) = long_option_action(word) { + return match action { + LongOptionAction::Consumed => Some(OptionAction::Consumed), + LongOptionAction::TakesNext => { + words.pop_front()?; + Some(OptionAction::Consumed) + } + LongOptionAction::Split(split) => expand_split(split, words, split_budget), + LongOptionAction::Invalid => None, + }; } - if let Some(split) = split_value(word) { + if let Some(split) = short_split_value(word) { return expand_split(split, words, split_budget); } match short_option_action(word) { @@ -109,52 +115,13 @@ fn expanded_words(first: &[u8], remaining: VecDeque>) -> Option Option<&[u8]> { - if word == b"-S" || word == b"--split-string" { +fn short_split_value(word: &[u8]) -> Option<&[u8]> { + if word == b"-S" { Some(b"") } else { - word.strip_prefix(b"-S") - .filter(|value| !value.is_empty()) - .or_else(|| word.strip_prefix(b"--split-string=")) + word.strip_prefix(b"-S").filter(|value| !value.is_empty()) } } -fn option_takes_value(word: &[u8]) -> bool { - matches!(word, b"--unset" | b"--chdir" | b"--argv0") -} - -fn option_has_value(word: &[u8]) -> bool { - [ - b"--unset=".as_slice(), - b"--chdir=".as_slice(), - b"--argv0=".as_slice(), - ] - .iter() - .any(|prefix| word.starts_with(prefix)) -} - -fn is_flag(word: &[u8]) -> bool { - matches!( - word, - b"--ignore-environment" - | b"--debug" - | b"--null" - | b"--help" - | b"--version" - | b"--list-signal-handling" - ) || [ - b"--block-signal".as_slice(), - b"--default-signal", - b"--ignore-signal", - ] - .iter() - .any(|prefix| { - word == *prefix - || word - .strip_prefix(*prefix) - .is_some_and(|value| value.starts_with(b"=")) - }) -} - #[cfg(test)] mod tests; diff --git a/xtask/src/source_structure/python_source/environment/long_options.rs b/xtask/src/source_structure/python_source/environment/long_options.rs new file mode 100644 index 0000000..3bc57bb --- /dev/null +++ b/xtask/src/source_structure/python_source/environment/long_options.rs @@ -0,0 +1,100 @@ +//! This module owns GNU `env` long-option abbreviation admission. + +#[derive(Clone, Copy)] +enum ValueMode { + None, + Required, + Optional, + Split, +} + +#[derive(Clone, Copy)] +struct LongOption { + name: &'static [u8], + value: ValueMode, +} + +struct LongOptionWord<'a> { + name: &'a [u8], + attached: Option<&'a [u8]>, +} + +const OPTIONS: [LongOption; 12] = [ + LongOption::new(b"--argv0", ValueMode::Required), + LongOption::new(b"--block-signal", ValueMode::Optional), + LongOption::new(b"--chdir", ValueMode::Required), + LongOption::new(b"--debug", ValueMode::None), + LongOption::new(b"--default-signal", ValueMode::Optional), + LongOption::new(b"--help", ValueMode::None), + LongOption::new(b"--ignore-environment", ValueMode::None), + LongOption::new(b"--ignore-signal", ValueMode::Optional), + LongOption::new(b"--list-signal-handling", ValueMode::None), + LongOption::new(b"--null", ValueMode::None), + LongOption::new(b"--split-string", ValueMode::Split), + LongOption::new(b"--unset", ValueMode::Required), +]; + +/// The effect of one admitted GNU `env` long option. +pub(super) enum LongOptionAction<'a> { + /// The option and any attached or optional value are complete. + Consumed, + /// The option requires the next word as its value. + TakesNext, + /// The option replaces the remaining words with split-string bytes. + Split(&'a [u8]), + /// The option forbids its attached value. + Invalid, +} + +/// Admits an exact or unambiguous abbreviated GNU `env` long option. +pub(super) fn action(word: &[u8]) -> Option> { + let word = split_value(word)?; + if !word.name.starts_with(b"--") { + return None; + } + let option = unique_match(word.name)?; + Some(match (option.value, word.attached) { + (ValueMode::None, None) | (ValueMode::Optional, _) | (ValueMode::Required, Some(_)) => { + LongOptionAction::Consumed + } + (ValueMode::Required, None) => LongOptionAction::TakesNext, + (ValueMode::Split, value) => LongOptionAction::Split(value.unwrap_or_default()), + (ValueMode::None, Some(_)) => LongOptionAction::Invalid, + }) +} + +fn unique_match(name: &[u8]) -> Option { + if let Some(option) = OPTIONS.iter().find(|option| option.name == name) { + return Some(*option); + } + let mut matches = OPTIONS + .iter() + .filter(|option| option.name.starts_with(name)) + .copied(); + let option = matches.next()?; + matches.next().is_none().then_some(option) +} + +fn split_value(word: &[u8]) -> Option> { + match word.iter().position(|byte| *byte == b'=') { + Some(index) => { + let name = word.get(..index)?; + let value_start = index.checked_add(1)?; + let value = word.get(value_start..)?; + Some(LongOptionWord { + name, + attached: Some(value), + }) + } + None => Some(LongOptionWord { + name: word, + attached: None, + }), + } +} + +impl LongOption { + const fn new(name: &'static [u8], value: ValueMode) -> Self { + Self { name, value } + } +} diff --git a/xtask/src/source_structure/python_source/environment/tests.rs b/xtask/src/source_structure/python_source/environment/tests.rs index 965a89e..478d956 100644 --- a/xtask/src/source_structure/python_source/environment/tests.rs +++ b/xtask/src/source_structure/python_source/environment/tests.rs @@ -41,3 +41,17 @@ fn unresolved_selected_utility_is_ambiguous() { Some(super::UtilitySelection::Ambiguous) ); } + +#[test] +fn unambiguous_long_option_abbreviations_preserve_the_utility() { + for arguments in [ + b"--spl=python3 -I".as_slice(), + b"--ignore-e python3", + b"--uns PYTHONHOME python3", + ] { + assert_eq!( + super::selected_utility(arguments), + Some(super::UtilitySelection::Known(b"python3".to_vec())) + ); + } +} From c8438edf557bc7b22a088fae972d736604937d5e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:16:37 -0700 Subject: [PATCH 096/113] Fix: retain fixed policy file identities --- CHANGELOG.md | 4 +- .../contributor_contract.rs | 3 +- .../src/documentation_integrity/dependabot.rs | 3 +- xtask/src/documentation_integrity/error.rs | 3 + .../documentation_integrity/error/display.rs | 7 ++ .../documentation_integrity/node_toolchain.rs | 7 +- .../repository_text.rs | 67 +++++++++++++++++-- .../repository_text/tests.rs | 26 ++++++- .../workflow_contract.rs | 3 +- 9 files changed, 111 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f111d5..3c2cad8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,9 @@ after its public API and format compatibility policies are established. bounds each admitted documentation source to 4 MiB and each selected corpus to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time - drift before and after each external tool, + drift before and after each external tool, retains each fixed policy file + identity through semantic admission so a replaced path cannot validate bytes + from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool execution, and output collection. Validation tools clear the inherited environment and admit only the executable search path and `C` locale, so diff --git a/xtask/src/documentation_integrity/contributor_contract.rs b/xtask/src/documentation_integrity/contributor_contract.rs index cf454f3..60f2316 100644 --- a/xtask/src/documentation_integrity/contributor_contract.rs +++ b/xtask/src/documentation_integrity/contributor_contract.rs @@ -14,7 +14,8 @@ const WHOLE_TREE_CHECK: &str = r#"git diff --check "$(git hash-object -t tree /d pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { for path in [CONTRIBUTING_PATH, STANDARDS_PATH] { let raw = repository_text::read(repository_root, path)?; - admit(path, &raw)?; + admit(path, raw.as_str())?; + raw.verify(repository_root)?; } Ok(()) } diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index f731bb1..fcef902 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -26,7 +26,8 @@ pub(super) fn check( ) -> Result<(), DocumentationError> { let raw = repository_text::read(repository_root, DEPENDABOT_PATH)?; let required = tracked_scopes(process_directory)?; - admit(&raw, &required) + admit(raw.as_str(), &required)?; + raw.verify(repository_root) } fn admit(raw: &str, required: &BTreeSet) -> Result<(), DocumentationError> { diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index ff076c5..5f46d50 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -71,6 +71,8 @@ pub(crate) enum DocumentationError { path: &'static str, source: io::Error, }, + /// A fixed policy path no longer names the file identity admitted for validation. + RepositoryFileChanged(&'static str), RepositoryFileNonRegular(&'static str), RepositoryFileTooLarge { path: &'static str, @@ -163,6 +165,7 @@ impl Error for DocumentationError { | Self::InvalidPath { .. } | Self::NonRegular { .. } | Self::RefusalMismatch { observed: None, .. } + | Self::RepositoryFileChanged(_) | Self::RepositoryFileNonRegular(_) | Self::RepositoryFileTooLarge { .. } | Self::RepositoryContract { .. } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index ca1ce6c..9018c39 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -60,6 +60,7 @@ impl fmt::Display for DocumentationError { refusal_mismatch(formatter, scenario, observed.as_deref()) } error @ (Self::RepositoryFileEncoding { .. } + | Self::RepositoryFileChanged(_) | Self::RepositoryFileInspect { .. } | Self::RepositoryFileNonRegular(_) | Self::RepositoryFileTooLarge { .. } @@ -116,6 +117,12 @@ fn corpus(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt fn repository_file(formatter: &mut fmt::Formatter<'_>, error: &DocumentationError) -> fmt::Result { match error { + DocumentationError::RepositoryFileChanged(path) => { + write!( + formatter, + "repository file changed during validation: `{path}`" + ) + } DocumentationError::RepositoryFileEncoding { path, .. } => { write!(formatter, "repository file `{path}` is not UTF-8") } diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index 3ac55d5..fadf2e7 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -25,8 +25,11 @@ pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), Documentatio let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; let lock = repository_text::read(repository_root, LOCK_PATH)?; let installer = repository_text::read(repository_root, INSTALLER_PATH)?; - admit(&manifest, &lock, &installer)?; - admit_lock_bytes(&lock) + admit(manifest.as_str(), lock.as_str(), installer.as_str())?; + admit_lock_bytes(lock.as_str())?; + manifest.verify(repository_root)?; + lock.verify(repository_root)?; + installer.verify(repository_root) } fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), DocumentationError> { diff --git a/xtask/src/documentation_integrity/repository_text.rs b/xtask/src/documentation_integrity/repository_text.rs index c82401e..c9bbb8b 100644 --- a/xtask/src/documentation_integrity/repository_text.rs +++ b/xtask/src/documentation_integrity/repository_text.rs @@ -1,21 +1,73 @@ //! This module owns bounded UTF-8 reads of fixed repository policy files. -use std::io::Read; +use std::io::{self, Read}; use std::path::Path; -use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; use super::error::DocumentationError; const MAX_REPOSITORY_FILE_BYTES: u64 = 1_048_576; +/// Bounded UTF-8 policy text with the file identity admitted for its bytes. +pub(super) struct RepositoryText { + identity: RepositoryFileIdentity, + path: &'static str, + text: String, +} + +impl RepositoryText { + /// Returns the admitted UTF-8 policy text. + pub(super) fn as_str(&self) -> &str { + &self.text + } + + /// Revalidates that the policy path still names the admitted file identity. + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + ) -> Result<(), DocumentationError> { + let current = match repository_root.open_file(Path::new(self.path)) { + Ok(file) => file, + Err(OpenRepositoryFileError::Io(source)) + if source.kind() == io::ErrorKind::NotFound => + { + return Err(DocumentationError::RepositoryFileChanged(self.path)); + } + Err(OpenRepositoryFileError::Io(source)) => { + return Err(DocumentationError::RepositoryFileInspect { + path: self.path, + source, + }); + } + Err(OpenRepositoryFileError::NonRegular) => { + return Err(DocumentationError::RepositoryFileChanged(self.path)); + } + }; + let current = RepositoryFileIdentity::read(¤t).map_err(|source| { + DocumentationError::RepositoryFileInspect { + path: self.path, + source, + } + })?; + if current == self.identity { + Ok(()) + } else { + Err(DocumentationError::RepositoryFileChanged(self.path)) + } + } +} + +/// Reads one fixed policy path through the retained repository authority. pub(super) fn read( repository_root: &RepositoryRoot, path: &'static str, -) -> Result { +) -> Result { let file = repository_root .open_file(Path::new(path)) .map_err(|error| open_error(path, error))?; + let identity = RepositoryFileIdentity::read(&file) + .map_err(|source| DocumentationError::RepositoryFileInspect { path, source })?; let read_bound = MAX_REPOSITORY_FILE_BYTES.checked_add(1).ok_or( DocumentationError::RepositoryFileTooLarge { path, @@ -32,8 +84,13 @@ pub(super) fn read( maximum: MAX_REPOSITORY_FILE_BYTES, }); } - String::from_utf8(bytes) - .map_err(|source| DocumentationError::RepositoryFileEncoding { path, source }) + let text = String::from_utf8(bytes) + .map_err(|source| DocumentationError::RepositoryFileEncoding { path, source })?; + Ok(RepositoryText { + identity, + path, + text, + }) } fn open_error(path: &'static str, error: OpenRepositoryFileError) -> DocumentationError { diff --git a/xtask/src/documentation_integrity/repository_text/tests.rs b/xtask/src/documentation_integrity/repository_text/tests.rs index 130a446..b17d99a 100644 --- a/xtask/src/documentation_integrity/repository_text/tests.rs +++ b/xtask/src/documentation_integrity/repository_text/tests.rs @@ -12,7 +12,7 @@ fn repository_policy_reads_are_utf8_and_bounded() -> Result<(), Box Result<(), Box Result<(), Box> { + let directory = TestDirectory::create("repository-text-identity")?; + let path = directory.path().join("policy.txt"); + let retained = directory.path().join("retained.txt"); + fs::write(&path, "admitted\n")?; + let root = RepositoryRoot::open(directory.path())?; + let policy = read(&root, "policy.txt")?; + + fs::rename(&path, &retained)?; + fs::write(&path, "replacement\n")?; + + assert!(matches!( + policy.verify(&root), + Err(super::DocumentationError::RepositoryFileChanged( + "policy.txt" + )) + )); + drop(root); + directory.close()?; + Ok(()) +} + #[test] fn repository_policy_reads_refuse_bytes_beyond_the_bound() -> Result<(), Box> { diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index dbe3897..434c2d5 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -23,7 +23,8 @@ const NODE_VERSION: &str = "24.18.0"; pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { let workflow = repository_text::read(repository_root, CI_PATH)?; - admit(&workflow) + admit(workflow.as_str())?; + workflow.verify(repository_root) } fn admit(workflow: &str) -> Result<(), DocumentationError> { From 89eacb75abf80fc374a49008ed8c14231bd6d35e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:19:06 -0700 Subject: [PATCH 097/113] Fix: revalidate documentation corpus membership --- CHANGELOG.md | 7 +++-- xtask/src/documentation_integrity/corpus.rs | 28 +++++++++++++++++++ .../corpus/replacement_tests.rs | 28 ++++++++++++++++++- .../src/documentation_integrity/execution.rs | 3 +- .../execution/corpus_guard.rs | 6 +++- .../execution/tests.rs | 3 +- 6 files changed, 68 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2cad8..07195f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,9 +38,10 @@ after its public API and format compatibility policies are established. bounds each admitted documentation source to 4 MiB and each selected corpus to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time - drift before and after each external tool, retains each fixed policy file - identity through semantic admission so a replaced path cannot validate bytes - from a superseded file, + drift before and after each external tool, re-inventories complete corpus + membership so newly added sources cannot bypass those tools, retains each + fixed policy file identity through semantic admission so a replaced path + cannot validate bytes from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool execution, and output collection. Validation tools clear the inherited environment and admit only the executable search path and `C` locale, so diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index 638ee80..802f6d4 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -6,6 +6,7 @@ mod source_witness; #[cfg(test)] pub(super) mod test_repository; +use std::cmp::Ordering; use std::io; use xtask::protocol_admission::posix_relative_path; @@ -47,10 +48,18 @@ impl SourceCorpus { pub(super) fn verify_unchanged( &self, repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, ) -> Result<(), DocumentationError> { for source in &self.sources { source.verify(repository_root, self.kind)?; } + let current = Self::read(repository_root, process_directory, self.kind)?; + if let Some(path) = membership_change(&self.paths, ¤t.paths) { + return Err(DocumentationError::CorpusChanged { + corpus: self.kind.label(), + path, + }); + } Ok(()) } @@ -87,6 +96,25 @@ impl SourceCorpus { } } +fn membership_change(expected: &[String], observed: &[String]) -> Option { + let mut expected = expected.iter().peekable(); + let mut observed = observed.iter().peekable(); + loop { + match (expected.peek(), observed.peek()) { + (Some(left), Some(right)) => match left.cmp(right) { + Ordering::Equal => { + expected.next(); + observed.next(); + } + Ordering::Less => return Some((*left).clone()), + Ordering::Greater => return Some((*right).clone()), + }, + (Some(path), None) | (None, Some(path)) => return Some((*path).clone()), + (None, None) => return None, + } + } +} + fn admit_paths<'a>( repository_root: &RepositoryRoot, paths: impl Iterator, diff --git a/xtask/src/documentation_integrity/corpus/replacement_tests.rs b/xtask/src/documentation_integrity/corpus/replacement_tests.rs index aaa133c..695b72a 100644 --- a/xtask/src/documentation_integrity/corpus/replacement_tests.rs +++ b/xtask/src/documentation_integrity/corpus/replacement_tests.rs @@ -21,7 +21,7 @@ fn selected_source_replacement_refuses_the_admitted_corpus() fs::rename(root.join("selected.md"), root.join("retained.md"))?; fs::write(root.join("selected.md"), "# Substitute\n")?; - let result = corpus.verify_unchanged(&repository_root); + let result = corpus.verify_unchanged(&repository_root, &process_directory); assert!(matches!( result, @@ -34,3 +34,29 @@ fn selected_source_replacement_refuses_the_admitted_corpus() directory.close()?; Ok(()) } + +#[test] +fn added_source_refuses_the_admitted_corpus() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-source-addition")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + fs::write(root.join("added.md"), "# Added\n")?; + + let result = corpus.verify_unchanged(&repository_root, &process_directory); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "added.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index c3ca313..68616e4 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -38,7 +38,8 @@ pub(super) fn run( ) -> Result<(), DocumentationError> { let corpora = [markdown, workflows]; let external = ExternalToolRunner { process_directory }; - let mut runner = CorpusGuardedRunner::new(external, repository_root, &corpora); + let mut runner = + CorpusGuardedRunner::new(external, process_directory, repository_root, &corpora); run_with(&mut runner, markdown.paths(), workflows.paths()) } diff --git a/xtask/src/documentation_integrity/execution/corpus_guard.rs b/xtask/src/documentation_integrity/execution/corpus_guard.rs index bec350c..eb924c5 100644 --- a/xtask/src/documentation_integrity/execution/corpus_guard.rs +++ b/xtask/src/documentation_integrity/execution/corpus_guard.rs @@ -3,6 +3,7 @@ use crate::bounded_process::ProcessOutput; use crate::documentation_integrity::corpus::SourceCorpus; use crate::documentation_integrity::error::DocumentationError; +use crate::repository_file::RepositoryProcessDirectory; use crate::repository_file::RepositoryRoot; use super::{DocumentationTool, ToolRunner}; @@ -10,25 +11,28 @@ use super::{DocumentationTool, ToolRunner}; pub(super) struct CorpusGuardedRunner<'a, Runner> { corpora: &'a [&'a SourceCorpus], inner: Runner, + process_directory: &'a RepositoryProcessDirectory, repository_root: &'a RepositoryRoot, } impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { pub(super) const fn new( inner: Runner, + process_directory: &'a RepositoryProcessDirectory, repository_root: &'a RepositoryRoot, corpora: &'a [&'a SourceCorpus], ) -> Self { Self { corpora, inner, + process_directory, repository_root, } } fn verify(&self) -> Result<(), DocumentationError> { for corpus in self.corpora { - corpus.verify_unchanged(self.repository_root)?; + corpus.verify_unchanged(self.repository_root, self.process_directory)?; } Ok(()) } diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index 62550bd..d70b7dd 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -164,7 +164,8 @@ fn corpus_guard_refuses_a_source_restored_after_transient_replacement() selected: root.join("selected.md"), retained: root.join("retained.md"), }; - let mut runner = CorpusGuardedRunner::new(replacing, &repository_root, &corpora); + let mut runner = + CorpusGuardedRunner::new(replacing, &process_directory, &repository_root, &corpora); let result = runner.capture(DocumentationTool::Markdownlint, &[]); From a90d56bd35243594332f41e57095d9a984e71fdf Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:31:17 -0700 Subject: [PATCH 098/113] Fix: bind documentation tools to admitted snapshots --- CHANGELOG.md | 4 +- xtask/src/documentation_integrity/corpus.rs | 10 +- .../corpus/source_witness.rs | 82 +++++- xtask/src/documentation_integrity/error.rs | 8 +- .../documentation_integrity/error/display.rs | 3 + .../src/documentation_integrity/execution.rs | 76 ++++-- .../execution/corpus_guard.rs | 27 +- .../execution/snapshot.rs | 249 ++++++++++++++++++ .../execution/tests.rs | 84 ++++-- 9 files changed, 481 insertions(+), 62 deletions(-) create mode 100644 xtask/src/documentation_integrity/execution/snapshot.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 07195f7..6cd86de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,9 @@ after its public API and format compatibility policies are established. to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time drift before and after each external tool, re-inventories complete corpus - membership so newly added sources cannot bypass those tools, retains each + membership so newly added sources cannot bypass those tools, executes the + tools against a private snapshot copied from the admitted source descriptors + so transient path substitution cannot redirect their reads, retains each fixed policy file identity through semantic admission so a replaced path cannot validate bytes from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index 802f6d4..bd17572 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -8,6 +8,7 @@ pub(super) mod test_repository; use std::cmp::Ordering; use std::io; +use std::path::Path; use xtask::protocol_admission::posix_relative_path; @@ -45,6 +46,13 @@ impl SourceCorpus { &self.paths } + pub(super) fn materialize(&self, snapshot_root: &Path) -> Result<(), DocumentationError> { + for source in &self.sources { + source.materialize(snapshot_root, self.kind)?; + } + Ok(()) + } + pub(super) fn verify_unchanged( &self, repository_root: &RepositoryRoot, @@ -147,7 +155,7 @@ fn admit_path( path: text.clone(), })?; match repository_root.open_file(&relative) { - Ok(file) => Ok(Some(AdmittedSource::admit(&file, text, relative, kind)?)), + Ok(file) => Ok(Some(AdmittedSource::admit(file, text, relative, kind)?)), Err(OpenRepositoryFileError::Io(source)) if source.kind() == io::ErrorKind::NotFound => { Ok(None) } diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs index e6ca187..e606939 100644 --- a/xtask/src/documentation_integrity/corpus/source_witness.rs +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -1,7 +1,10 @@ //! This module owns retained identity evidence for one documentation source. use std::fs::File; -use std::io; +use std::fs::{self, OpenOptions}; +use std::io::{self, Write}; +use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::path::Path; use std::path::PathBuf; use super::CorpusKind; @@ -9,6 +12,7 @@ use crate::documentation_integrity::error::DocumentationError; use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; pub(super) struct AdmittedSource { + file: File, identity: RepositoryFileIdentity, path: String, relative: PathBuf, @@ -16,13 +20,14 @@ pub(super) struct AdmittedSource { impl AdmittedSource { pub(super) fn admit( - file: &File, + file: File, path: String, relative: PathBuf, kind: CorpusKind, ) -> Result { - let identity = identity(file, kind, &path)?; + let identity = identity(&file, kind, &path)?; Ok(Self { + file, identity, path, relative, @@ -37,6 +42,32 @@ impl AdmittedSource { &self.path } + pub(super) fn materialize( + &self, + snapshot_root: &Path, + kind: CorpusKind, + ) -> Result<(), DocumentationError> { + self.verify_retained(kind)?; + let destination = snapshot_root.join(&self.relative); + let parent = destination.parent().ok_or_else(|| { + snapshot_io( + "resolve documentation snapshot parent", + io::Error::other("source path has no parent"), + ) + })?; + fs::create_dir_all(parent) + .map_err(|source| snapshot_io("create documentation snapshot directory", source))?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(&destination) + .map_err(|source| snapshot_io("create documentation snapshot source", source))?; + copy_exact(&self.file, &mut output, self.identity.bytes()) + .map_err(|source| snapshot_io("copy documentation snapshot source", source))?; + self.verify_retained(kind) + } + pub(super) fn verify( &self, repository_root: &RepositoryRoot, @@ -67,6 +98,47 @@ impl AdmittedSource { Err(changed(kind, &self.path)) } } + + fn verify_retained(&self, kind: CorpusKind) -> Result<(), DocumentationError> { + let current = identity(&self.file, kind, &self.path)?; + if current == self.identity { + Ok(()) + } else { + Err(changed(kind, &self.path)) + } + } +} + +fn copy_exact(source: &File, destination: &mut File, expected: u64) -> Result<(), io::Error> { + let mut offset = 0_u64; + let mut buffer = [0_u8; 16_384]; + while offset < expected { + let remaining = expected + .checked_sub(offset) + .ok_or_else(|| io::Error::other("snapshot source offset exceeded its length"))?; + let limit = + usize::try_from(remaining).map_or(buffer.len(), |bytes| bytes.min(buffer.len())); + let chunk = buffer + .get_mut(..limit) + .ok_or_else(|| io::Error::other("snapshot read bound exceeded its buffer"))?; + let read = source.read_at(chunk, offset)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "snapshot source ended before its admitted length", + )); + } + let copied = buffer + .get(..read) + .ok_or_else(|| io::Error::other("snapshot write bound exceeded its buffer"))?; + destination.write_all(copied)?; + offset = offset + .checked_add(u64::try_from(read).map_err(|_| { + io::Error::other("snapshot source read length is not representable") + })?) + .ok_or_else(|| io::Error::other("snapshot source offset overflowed"))?; + } + Ok(()) } fn identity( @@ -87,3 +159,7 @@ fn changed(kind: CorpusKind, path: &str) -> DocumentationError { path: path.to_owned(), } } + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 5f46d50..8ec7f03 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -106,6 +106,11 @@ pub(crate) enum DocumentationError { expected: &'static str, observed: Option, }, + /// A filesystem operation could not construct or remove a tool-input snapshot. + Snapshot { + action: &'static str, + source: io::Error, + }, VersionMismatch { program: &'static str, expected: &'static str, @@ -144,7 +149,8 @@ impl Error for DocumentationError { Self::Inspect { source, .. } | Self::RefusalFixture { source, .. } | Self::RepositoryFileInspect { source, .. } - | Self::RepositoryRootInspect { source, .. } => Some(source), + | Self::RepositoryRootInspect { source, .. } + | Self::Snapshot { source, .. } => Some(source), Self::PathEncoding { source, .. } | Self::RepositoryFileEncoding { source, .. } => { Some(source) } diff --git a/xtask/src/documentation_integrity/error/display.rs b/xtask/src/documentation_integrity/error/display.rs index 9018c39..5e675bd 100644 --- a/xtask/src/documentation_integrity/error/display.rs +++ b/xtask/src/documentation_integrity/error/display.rs @@ -75,6 +75,9 @@ impl fmt::Display for DocumentationError { Self::RepositoryRootInspect { path, .. } => { repository_root(formatter, RepositoryRootDiagnostic::Inspect, path) } + Self::Snapshot { action, .. } => { + write!(formatter, "cannot {action}") + } error @ (Self::VersionMismatch { .. } | Self::ToolFailed { .. } | Self::ToolOutputEncoding { .. } diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index 68616e4..6257758 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -2,6 +2,7 @@ mod corpus_guard; mod refusal_check; +mod snapshot; use std::env; use std::ffi::OsStr; @@ -26,6 +27,16 @@ trait ToolRunner { ) -> Result; } +trait DirectoryToolRunner { + fn capture_in( + &mut self, + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], + ) -> Result; +} + struct ExternalToolRunner<'a> { process_directory: &'a RepositoryProcessDirectory, } @@ -39,8 +50,11 @@ pub(super) fn run( let corpora = [markdown, workflows]; let external = ExternalToolRunner { process_directory }; let mut runner = - CorpusGuardedRunner::new(external, process_directory, repository_root, &corpora); - run_with(&mut runner, markdown.paths(), workflows.paths()) + CorpusGuardedRunner::new(external, process_directory, repository_root, &corpora)?; + let result = run_with(&mut runner, markdown.paths(), workflows.paths()); + let cleanup = runner.close(); + result?; + cleanup } /// Executes both named malformed-input scenarios through the production runner. @@ -155,28 +169,48 @@ impl ToolRunner for ExternalToolRunner<'_> { tool: DocumentationTool, arguments: &[String], ) -> Result { - let path = env::var_os("PATH").ok_or(DocumentationError::EnvironmentUnavailable("PATH"))?; - let mut command = documentation_command(tool, arguments, &path); - bounded_process::capture_with( - tool.program(), - &mut command, - Some(TOOL_DEADLINE), - |command| self.process_directory.spawn(command), - ) - .map_err(|source| { - if source.is_not_found() { - DocumentationError::ToolUnavailable { - program: tool.program(), - install_version: tool.install_version(), - source, - } - } else { - DocumentationError::Process(source) - } - }) + capture_external(self.process_directory, tool, arguments) + } +} + +impl DirectoryToolRunner for ExternalToolRunner<'_> { + fn capture_in( + &mut self, + _repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], + ) -> Result { + capture_external(process_directory, tool, arguments) } } +fn capture_external( + process_directory: &RepositoryProcessDirectory, + tool: DocumentationTool, + arguments: &[String], +) -> Result { + let path = env::var_os("PATH").ok_or(DocumentationError::EnvironmentUnavailable("PATH"))?; + let mut command = documentation_command(tool, arguments, &path); + bounded_process::capture_with( + tool.program(), + &mut command, + Some(TOOL_DEADLINE), + |command| process_directory.spawn(command), + ) + .map_err(|source| { + if source.is_not_found() { + DocumentationError::ToolUnavailable { + program: tool.program(), + install_version: tool.install_version(), + source, + } + } else { + DocumentationError::Process(source) + } + }) +} + fn documentation_command(tool: DocumentationTool, arguments: &[String], path: &OsStr) -> Command { let mut command = Command::new(tool.program()); command diff --git a/xtask/src/documentation_integrity/execution/corpus_guard.rs b/xtask/src/documentation_integrity/execution/corpus_guard.rs index eb924c5..1c7c675 100644 --- a/xtask/src/documentation_integrity/execution/corpus_guard.rs +++ b/xtask/src/documentation_integrity/execution/corpus_guard.rs @@ -6,28 +6,36 @@ use crate::documentation_integrity::error::DocumentationError; use crate::repository_file::RepositoryProcessDirectory; use crate::repository_file::RepositoryRoot; -use super::{DocumentationTool, ToolRunner}; +use super::snapshot::DocumentationSnapshot; +use super::{DirectoryToolRunner, DocumentationTool, ToolRunner}; pub(super) struct CorpusGuardedRunner<'a, Runner> { corpora: &'a [&'a SourceCorpus], inner: Runner, process_directory: &'a RepositoryProcessDirectory, repository_root: &'a RepositoryRoot, + snapshot: DocumentationSnapshot, } impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { - pub(super) const fn new( + pub(super) fn new( inner: Runner, process_directory: &'a RepositoryProcessDirectory, repository_root: &'a RepositoryRoot, corpora: &'a [&'a SourceCorpus], - ) -> Self { - Self { + ) -> Result { + let snapshot = DocumentationSnapshot::create(repository_root, process_directory, corpora)?; + Ok(Self { corpora, inner, process_directory, repository_root, - } + snapshot, + }) + } + + pub(super) fn close(self) -> Result<(), DocumentationError> { + self.snapshot.close() } fn verify(&self) -> Result<(), DocumentationError> { @@ -40,7 +48,7 @@ impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { impl ToolRunner for CorpusGuardedRunner<'_, Runner> where - Runner: ToolRunner, + Runner: DirectoryToolRunner, { fn capture( &mut self, @@ -48,7 +56,12 @@ where arguments: &[String], ) -> Result { self.verify()?; - let result = self.inner.capture(tool, arguments); + let result = self.inner.capture_in( + self.snapshot.repository_root(), + self.snapshot.process_directory(), + tool, + arguments, + ); self.verify()?; result } diff --git a/xtask/src/documentation_integrity/execution/snapshot.rs b/xtask/src/documentation_integrity/execution/snapshot.rs new file mode 100644 index 0000000..d9e07fb --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot.rs @@ -0,0 +1,249 @@ +//! This module owns immutable documentation-tool input snapshots. + +use std::collections::BTreeSet; +use std::fs::{self, DirBuilder, OpenOptions}; +use std::io::{self, Write}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use xtask::protocol_admission::posix_relative_path; + +use crate::documentation_integrity::corpus::SourceCorpus; +use crate::documentation_integrity::error::DocumentationError; +use crate::documentation_integrity::repository_text::{self, RepositoryText}; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +const CREATION_ATTEMPTS: u16 = 1_024; +const MARKDOWNLINT_CONFIG: &str = ".markdownlint-cli2.yaml"; +const NAMESPACE_PRESENT: [&str; 5] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", +]; +const NAMESPACE_DELETED: [&str; 3] = ["ls-files", "-z", "--deleted"]; +static NEXT_SNAPSHOT: AtomicU64 = AtomicU64::new(0); + +pub(super) struct DocumentationSnapshot { + directory: SnapshotDirectory, + process_directory: RepositoryProcessDirectory, + repository_root: RepositoryRoot, +} + +impl DocumentationSnapshot { + pub(super) fn create( + source_root: &RepositoryRoot, + source_process_directory: &RepositoryProcessDirectory, + corpora: &[&SourceCorpus], + ) -> Result { + let config = repository_text::read(source_root, MARKDOWNLINT_CONFIG)?; + let directory = SnapshotDirectory::create()?; + let materialized = materialize(directory.path(), corpora, &config)?; + materialize_namespace(directory.path(), source_process_directory, &materialized)?; + verify_sources(source_root, source_process_directory, corpora, &config)?; + let repository_root = open_snapshot_root(directory.path())?; + let process_directory = repository_root.process_directory().map_err(|source| { + snapshot_io("open documentation snapshot process directory", source) + })?; + Ok(Self { + directory, + process_directory, + repository_root, + }) + } + + pub(super) const fn process_directory(&self) -> &RepositoryProcessDirectory { + &self.process_directory + } + + pub(super) const fn repository_root(&self) -> &RepositoryRoot { + &self.repository_root + } + + pub(super) fn close(self) -> Result<(), DocumentationError> { + let Self { + directory, + process_directory, + repository_root, + } = self; + drop(process_directory); + drop(repository_root); + directory + .close() + .map_err(|source| snapshot_io("remove documentation snapshot", source)) + } +} + +fn materialize( + destination: &Path, + corpora: &[&SourceCorpus], + config: &RepositoryText, +) -> Result, DocumentationError> { + let mut materialized = BTreeSet::new(); + for corpus in corpora { + corpus.materialize(destination)?; + for path in corpus.paths() { + materialized.insert(PathBuf::from(path)); + } + } + write_config(destination, config)?; + materialized.insert(PathBuf::from(MARKDOWNLINT_CONFIG)); + Ok(materialized) +} + +fn write_config(destination: &Path, config: &RepositoryText) -> Result<(), DocumentationError> { + let path = destination.join(MARKDOWNLINT_CONFIG); + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(path) + .map_err(|source| snapshot_io("create documentation snapshot configuration", source))?; + output + .write_all(config.as_str().as_bytes()) + .map_err(|source| snapshot_io("write documentation snapshot configuration", source)) +} + +fn materialize_namespace( + destination: &Path, + process_directory: &RepositoryProcessDirectory, + materialized: &BTreeSet, +) -> Result<(), DocumentationError> { + let present = paths_with( + &NAMESPACE_PRESENT, + "git documentation snapshot present paths", + |command| process_directory.spawn(command), + )?; + let deleted = paths_with( + &NAMESPACE_DELETED, + "git documentation snapshot deleted paths", + |command| process_directory.spawn(command), + )?; + for path in present.difference(&deleted) { + if let Some(relative) = snapshot_relative(path)? + && !materialized.contains(&relative) + { + create_placeholder(destination, &relative)?; + } + } + Ok(()) +} + +fn snapshot_relative(path: &GitPath) -> Result, DocumentationError> { + let Ok(text) = std::str::from_utf8(path.as_bytes()) else { + return Ok(None); + }; + posix_relative_path(text) + .map(Some) + .map_err(|_| DocumentationError::InvalidPath { + corpus: "documentation snapshot namespace", + path: text.to_owned(), + }) +} + +fn create_placeholder(destination: &Path, relative: &Path) -> Result<(), DocumentationError> { + let path = destination.join(relative); + let parent = path.parent().ok_or_else(|| { + snapshot_io( + "resolve documentation snapshot namespace parent", + io::Error::other("namespace path has no parent"), + ) + })?; + fs::create_dir_all(parent) + .map_err(|source| snapshot_io("create documentation snapshot namespace", source))?; + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(path) + .map(|_| ()) + .map_err(|source| snapshot_io("create documentation snapshot placeholder", source)) +} + +fn verify_sources( + source_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + corpora: &[&SourceCorpus], + config: &RepositoryText, +) -> Result<(), DocumentationError> { + for corpus in corpora { + corpus.verify_unchanged(source_root, process_directory)?; + } + config.verify(source_root) +} + +fn open_snapshot_root(path: &Path) -> Result { + RepositoryRoot::open(path) + .map_err(|source| snapshot_io("open documentation snapshot root", source)) +} + +struct SnapshotDirectory { + path: PathBuf, + active: bool, +} + +impl SnapshotDirectory { + fn create() -> Result { + for _ in 0_u16..CREATION_ATTEMPTS { + let sequence = next_sequence()?; + let path = std::env::temp_dir().join(format!( + "keep-documentation-snapshot-{}-{sequence}", + std::process::id() + )); + let mut builder = DirBuilder::new(); + builder.mode(0o700); + match builder.create(&path) { + Ok(()) => return Ok(Self { path, active: true }), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(snapshot_io("create documentation snapshot", source)); + } + } + } + Err(snapshot_io( + "create documentation snapshot", + io::Error::new( + io::ErrorKind::AlreadyExists, + "documentation snapshot collision bound exhausted", + ), + )) + } + + fn path(&self) -> &Path { + self.path.as_path() + } + + fn close(mut self) -> Result<(), io::Error> { + fs::remove_dir_all(&self.path)?; + self.active = false; + Ok(()) + } +} + +impl Drop for SnapshotDirectory { + fn drop(&mut self) { + if self.active { + drop(fs::remove_dir_all(&self.path)); + } + } +} + +fn next_sequence() -> Result { + NEXT_SNAPSHOT + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_add(1) + }) + .map_err(|_| { + snapshot_io( + "allocate documentation snapshot identity", + io::Error::other("documentation snapshot sequence exhausted"), + ) + }) +} + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index d70b7dd..2f171eb 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -1,7 +1,8 @@ use std::collections::{BTreeMap, VecDeque}; use std::ffi::OsString; use std::fs; -use std::path::PathBuf; +use std::io::Read; +use std::path::{Path, PathBuf}; use crate::bounded_process::ProcessOutput; use crate::documentation_integrity::corpus::{SourceCorpus, test_repository::run_git}; @@ -9,7 +10,9 @@ use crate::repository_file::RepositoryRoot; use crate::test_directory::TestDirectory; use super::corpus_guard::CorpusGuardedRunner; -use super::{DocumentationError, DocumentationTool, ToolRunner, documentation_command}; +use super::{ + DirectoryToolRunner, DocumentationError, DocumentationTool, ToolRunner, documentation_command, +}; struct RecordingRunner { calls: Vec<(DocumentationTool, Vec)>, @@ -17,8 +20,7 @@ struct RecordingRunner { } struct ReplacingRunner { - selected: PathBuf, - retained: PathBuf, + repository: PathBuf, } #[test] @@ -150,33 +152,31 @@ fn unreviewed_version_stops_before_tool_execution() { } #[test] -fn corpus_guard_refuses_a_source_restored_after_transient_replacement() +fn corpus_guard_executes_against_the_admitted_source_snapshot() -> Result<(), Box> { let directory = TestDirectory::create("documentation-transient-source")?; let root = directory.path(); run_git(root, &["init", "--quiet", "--template="])?; - fs::write(root.join("selected.md"), "# Original\n")?; + fs::create_dir(root.join("docs"))?; + fs::write(root.join("docs/selected.md"), "# Original\n")?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; let repository_root = RepositoryRoot::open(root)?; let process_directory = repository_root.process_directory()?; let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; let corpora = [&corpus]; let replacing = ReplacingRunner { - selected: root.join("selected.md"), - retained: root.join("retained.md"), + repository: root.to_owned(), }; let mut runner = - CorpusGuardedRunner::new(replacing, &process_directory, &repository_root, &corpora); + CorpusGuardedRunner::new(replacing, &process_directory, &repository_root, &corpora)?; - let result = runner.capture(DocumentationTool::Markdownlint, &[]); + let output = runner.capture(DocumentationTool::Markdownlint, &[])?; - assert!(matches!( - result, - Err(DocumentationError::CorpusChanged { - corpus: "Markdown", - ref path, - }) if path == "selected.md" - )); - drop(runner); + assert_eq!(output.stdout, b"# Original\n"); + runner.close()?; drop(corpus); directory.close()?; Ok(()) @@ -207,24 +207,52 @@ impl ToolRunner for RecordingRunner { } } -impl ToolRunner for ReplacingRunner { - fn capture( +impl DirectoryToolRunner for ReplacingRunner { + fn capture_in( &mut self, + execution_root: &RepositoryRoot, + _process_directory: &crate::repository_file::RepositoryProcessDirectory, _tool: DocumentationTool, _arguments: &[String], ) -> Result { - fs::rename(&self.selected, &self.retained) - .map_err(|source| fixture_io("retain selected source", source))?; - fs::write(&self.selected, "# Substitute\n") + let selected = self.repository.join("docs"); + let retained = self.repository.join("retained-docs"); + fs::rename(&selected, &retained) + .map_err(|source| fixture_io("retain selected directory", source))?; + fs::create_dir(&selected) + .map_err(|source| fixture_io("create substitute directory", source))?; + fs::write(selected.join("selected.md"), "# Substitute\n") .map_err(|source| fixture_io("write substitute source", source))?; - fs::remove_file(&self.selected) - .map_err(|source| fixture_io("remove substitute source", source))?; - fs::rename(&self.retained, &self.selected) - .map_err(|source| fixture_io("restore selected source", source))?; - Ok(success()) + let observed = read_source(execution_root, Path::new("docs/selected.md"))?; + fs::remove_dir_all(&selected) + .map_err(|source| fixture_io("remove substitute directory", source))?; + fs::rename(&retained, &selected) + .map_err(|source| fixture_io("restore selected directory", source))?; + Ok(ProcessOutput { + code: Some(0), + succeeded: true, + stdout: observed, + stderr: Vec::new(), + }) } } +fn read_source( + repository_root: &RepositoryRoot, + relative: &Path, +) -> Result, DocumentationError> { + let mut file = repository_root.open_file(relative).map_err(|_| { + fixture_io( + "open execution source", + std::io::Error::other("open failed"), + ) + })?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|source| fixture_io("read execution source", source))?; + Ok(bytes) +} + fn fixture_io(requirement: &'static str, source: std::io::Error) -> DocumentationError { DocumentationError::Inspect { corpus: "test", From 0bd700ef7971f9354f4d12a02625c790dca6e709 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:36:51 -0700 Subject: [PATCH 099/113] Fix: reconcile tracked executable modes --- CHANGELOG.md | 12 ++-- xtask/src/source_structure.rs | 31 +++++++-- xtask/src/source_structure/pure_rust_tests.rs | 59 +++++++++++++++++ xtask/src/source_structure/source_error.rs | 25 +++++++- xtask/src/source_structure/source_file.rs | 48 ++++++++++++++ .../src/source_structure/source_inventory.rs | 63 ++++++++++++++++++- .../src/source_structure/tests/replacement.rs | 20 ++++-- 7 files changed, 241 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd86de..a935992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,11 +115,13 @@ after its public API and format compatibility policies are established. shebangs parse exact and unambiguous abbreviated long options, combined short-option clusters, assignments, quoting, and split strings before classifying only the selected utility, so later command arguments cannot - impersonate Python and unresolved utility substitutions fail closed. Source - execution, shebang, and physical-line evidence now come from one admitted - file descriptor whose identity is revalidated after each read phase, so path - replacement or in-place mutation cannot splice different file states into - one verification result. + impersonate Python and unresolved utility substitutions fail closed. Tracked + file modes are admitted from the Git index and must agree with the worktree, + so a staged executable cannot defer Python screening until the next checkout. + Source execution, shebang, and physical-line evidence now come from one + admitted file descriptor whose identity is revalidated after each read phase, + so path replacement or in-place mutation cannot splice different file states + into one verification result. - Git path inventory failures now remain primary when child cleanup, waiting, or diagnostic collection also fails; the secondary failure remains typed and inspectable. Empty path records and unterminated path bytes produce distinct, diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 2912917..7b485a5 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -7,13 +7,14 @@ mod source_file; mod source_inventory; mod source_kind; +use std::collections::BTreeMap; use std::io::{self, BufRead, BufReader}; use std::path::Path; use crate::repository_file::RepositoryRoot; use repository_path::RepositoryPath; pub(super) use source_error::SourceStructureError; -use source_file::{AdmittedSource, FileExecution, SourceFileAdmission}; +use source_file::{AdmittedSource, FileExecution, SourceFileAdmission, TrackedFileMode}; #[cfg(test)] use source_inventory::{ PRESENT_PATH_ARGUMENTS, select as select_source_inventory, select_source_paths, @@ -71,10 +72,16 @@ fn inventory_violations( source_root: &RepositoryRoot, inventory: SourceInventory, ) -> Result, SourceStructureError> { + let SourceInventory { + modules, + executable_candidates, + tracked_modes, + } = inventory; let mut violations = Vec::new(); - for relative in inventory.executable_candidates { + for relative in executable_candidates { + let tracked_mode = tracked_modes.get(relative.as_path()).copied(); let SourceFileAdmission::Regular(source) = - AdmittedSource::admit(source_root, relative.as_path())? + AdmittedSource::admit(source_root, relative.as_path(), tracked_mode)? else { return Err(SourceStructureError::NonRegular( source_root.display_path(relative.as_path()), @@ -86,19 +93,33 @@ fn inventory_violations( violations.push(relative.as_path().to_owned()); } } - violations.extend(source_violations(source_root, inventory.modules)?); + violations.extend(source_violations_with_modes( + source_root, + modules, + &tracked_modes, + )?); violations.sort(); Ok(violations) } +#[cfg(test)] fn source_violations( source_root: &RepositoryRoot, paths: Vec, +) -> Result, SourceStructureError> { + source_violations_with_modes(source_root, paths, &BTreeMap::new()) +} + +fn source_violations_with_modes( + source_root: &RepositoryRoot, + paths: Vec, + tracked_modes: &BTreeMap, ) -> Result, SourceStructureError> { let mut violations = Vec::new(); for relative in paths { + let tracked_mode = tracked_modes.get(relative.as_path()).copied(); let SourceFileAdmission::Regular(source) = - AdmittedSource::admit(source_root, relative.as_path())? + AdmittedSource::admit(source_root, relative.as_path(), tracked_mode)? else { return Err(SourceStructureError::NonRegular( source_root.display_path(relative.as_path()), diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 80d1b87..08e56a1 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -4,7 +4,10 @@ use std::collections::BTreeSet; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::time::Duration; +use crate::bounded_process; use crate::git_inventory::GitPath; use crate::repository_file::RepositoryRoot; use crate::test_directory::TestDirectory; @@ -117,6 +120,33 @@ fn extensionless_nonexecutable_text_is_not_a_source_module() Ok(()) } +#[test] +fn tracked_executable_mode_must_match_the_worktree() -> Result<(), Box> { + let directory = TestDirectory::create("tracked-executable-mode")?; + let repository = directory.path().join("repository"); + fs::create_dir(&repository)?; + run_git(&repository, &["init", "--quiet", "--template="])?; + fs::write( + repository.join("script"), + "#!/usr/bin/env python3\nprint('forbidden')\n", + )?; + run_git(&repository, &["add", "script"])?; + run_git(&repository, &["update-index", "--chmod=+x", "script"])?; + + let result = super::check(&repository); + + assert!(matches!( + result, + Err(super::SourceStructureError::ExecutionModeChanged { + ref path, + tracked: "executable", + worktree: "nonexecutable", + }) if path == &repository.join("script") + )); + directory.close()?; + Ok(()) +} + #[derive(Clone, Copy)] enum FixtureMode { Executable, @@ -159,3 +189,32 @@ impl SourceFixture { Ok(()) } } + +fn run_git( + repository: &std::path::Path, + arguments: &[&str], +) -> Result<(), Box> { + let path = std::env::var_os("PATH").ok_or("test PATH is unavailable")?; + let mut command = Command::new("git"); + command + .args(arguments) + .current_dir(repository) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .env_clear() + .env("PATH", path) + .env("LC_ALL", "C") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null"); + let output = bounded_process::status( + "git test fixture", + &mut command, + Some(Duration::from_secs(10)), + )?; + if output.succeeded { + Ok(()) + } else { + Err(format!("git test fixture failed with status {:?}", output.code).into()) + } +} diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 2ca7570..4992b18 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -10,7 +10,13 @@ use crate::diagnostic::{escaped_controls, escaped_path}; use crate::git_inventory::GitInventoryError; pub(crate) enum SourceStructureError { + ExecutionModeChanged { + path: PathBuf, + tracked: &'static str, + worktree: &'static str, + }, GitInventory(GitInventoryError), + GitIndexRecord, GitPathEncoding { operation: &'static str, source: FromUtf8Error, @@ -39,7 +45,22 @@ impl fmt::Debug for SourceStructureError { impl fmt::Display for SourceStructureError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::ExecutionModeChanged { + path, + tracked, + worktree, + } => { + formatter.write_str("repository execution mode differs from the index for `")?; + escaped_path(formatter, path)?; + write!( + formatter, + "`: tracked mode is {tracked}, worktree mode is {worktree}" + ) + } Self::GitInventory(error) => write!(formatter, "{error}"), + Self::GitIndexRecord => { + formatter.write_str("git returned an invalid tracked file-mode record") + } Self::GitPathEncoding { operation, .. } => { write!(formatter, "`{operation}` returned a non-UTF-8 path") } @@ -84,7 +105,9 @@ impl Error for SourceStructureError { Self::GitInventory(error) => Some(error), Self::GitPathEncoding { source, .. } => Some(source), Self::Inspect { source, .. } => Some(source), - Self::InvalidPath(_) + Self::ExecutionModeChanged { .. } + | Self::GitIndexRecord + | Self::InvalidPath(_) | Self::NonRegular(_) | Self::PythonSource(_) | Self::RepositoryRootChanged(_) diff --git a/xtask/src/source_structure/source_file.rs b/xtask/src/source_structure/source_file.rs index 3123da3..866e7b1 100644 --- a/xtask/src/source_structure/source_file.rs +++ b/xtask/src/source_structure/source_file.rs @@ -16,6 +16,12 @@ pub(super) enum FileExecution { NonExecutable, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum TrackedFileMode { + NonRegular, + Regular(FileExecution), +} + pub(super) enum SourceFileAdmission { Regular(AdmittedSource), NonRegular, @@ -33,11 +39,15 @@ impl AdmittedSource { pub(super) fn admit( source_root: &RepositoryRoot, relative: &Path, + tracked_execution: Option, ) -> Result { let path = source_root.display_path(relative); let file = match source_root.open_file(relative) { Ok(file) => file, Err(OpenRepositoryFileError::NonRegular) => { + if let Some(TrackedFileMode::Regular(execution)) = tracked_execution { + return Err(mode_changed(&path, execution.label(), "nonregular")); + } return Ok(SourceFileAdmission::NonRegular); } Err(OpenRepositoryFileError::Io(source)) => { @@ -51,6 +61,7 @@ impl AdmittedSource { source, })?; let execution = file_execution(&metadata); + admit_execution(&path, tracked_execution, execution)?; let python = execution == FileExecution::Executable && executable_uses_python(&file).map_err(|source| SourceStructureError::Inspect { path: path.clone(), @@ -116,6 +127,43 @@ impl AdmittedSource { } } +fn admit_execution( + path: &Path, + tracked: Option, + worktree: FileExecution, +) -> Result<(), SourceStructureError> { + match tracked { + Some(TrackedFileMode::Regular(tracked)) if tracked != worktree => { + Err(mode_changed(path, tracked.label(), worktree.label())) + } + Some(TrackedFileMode::NonRegular) => { + Err(mode_changed(path, "nonregular", worktree.label())) + } + Some(TrackedFileMode::Regular(_)) | None => Ok(()), + } +} + +fn mode_changed( + path: &Path, + tracked: &'static str, + worktree: &'static str, +) -> SourceStructureError { + SourceStructureError::ExecutionModeChanged { + path: path.to_owned(), + tracked, + worktree, + } +} + +impl FileExecution { + const fn label(self) -> &'static str { + match self { + Self::Executable => "executable", + Self::NonExecutable => "nonexecutable", + } + } +} + fn file_execution(metadata: &Metadata) -> FileExecution { if metadata.permissions().mode() & 0o111 == 0 { FileExecution::NonExecutable diff --git a/xtask/src/source_structure/source_inventory.rs b/xtask/src/source_structure/source_inventory.rs index 90d2cba..9105b5b 100644 --- a/xtask/src/source_structure/source_inventory.rs +++ b/xtask/src/source_structure/source_inventory.rs @@ -1,6 +1,6 @@ //! This module owns deterministic source and executable-candidate inventory. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::ffi::OsString; use std::os::unix::ffi::OsStringExt; use std::path::{Component, Path, PathBuf}; @@ -10,6 +10,7 @@ use crate::repository_file::RepositoryProcessDirectory; use super::repository_path::RepositoryPath; use super::source_error::SourceStructureError; +use super::source_file::{FileExecution, TrackedFileMode}; use super::source_kind::{is_python_module, is_source_candidate}; pub(super) const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ @@ -23,6 +24,7 @@ pub(super) const PRESENT_PATH_ARGUMENTS: [&str; 5] = [ pub(super) struct SourceInventory { pub(super) modules: Vec, pub(super) executable_candidates: Vec, + pub(super) tracked_modes: BTreeMap, } /// A repository-relative path admitted for executable inspection. @@ -46,7 +48,10 @@ pub(super) fn collect( "git ls-files deleted", |command| process_directory.spawn(command), )?; - select(&present, &deleted) + let tracked_modes = tracked_modes(process_directory)?; + let mut inventory = select(&present, &deleted)?; + inventory.tracked_modes = tracked_modes; + Ok(inventory) } pub(super) fn select( @@ -65,9 +70,63 @@ pub(super) fn select( Ok(SourceInventory { modules, executable_candidates, + tracked_modes: BTreeMap::new(), }) } +fn tracked_modes( + process_directory: &RepositoryProcessDirectory, +) -> Result, SourceStructureError> { + let records = paths_with( + &["ls-files", "-z", "--cached", "--stage"], + "git ls-files tracked modes", + |command| process_directory.spawn(command), + )?; + let mut modes = BTreeMap::new(); + for record in records { + let (path, mode) = parse_tracked_mode(&record)?; + if modes.insert(path, mode).is_some() { + return Err(SourceStructureError::GitIndexRecord); + } + } + Ok(modes) +} + +fn parse_tracked_mode( + record: &GitPath, +) -> Result<(PathBuf, TrackedFileMode), SourceStructureError> { + let bytes = record.as_bytes(); + let tab = bytes + .iter() + .position(|byte| *byte == b'\t') + .ok_or(SourceStructureError::GitIndexRecord)?; + let path_start = tab + .checked_add(1) + .ok_or(SourceStructureError::GitIndexRecord)?; + let header = bytes + .get(..tab) + .ok_or(SourceStructureError::GitIndexRecord)?; + let path = bytes + .get(path_start..) + .filter(|path| !path.is_empty()) + .ok_or(SourceStructureError::GitIndexRecord)?; + let mut fields = header.split(|byte| *byte == b' '); + let mode = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + let object = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + let stage = fields.next().ok_or(SourceStructureError::GitIndexRecord)?; + if object.is_empty() || stage != b"0" || fields.next().is_some() { + return Err(SourceStructureError::GitIndexRecord); + } + let mode = match mode { + b"100644" => TrackedFileMode::Regular(FileExecution::NonExecutable), + b"100755" => TrackedFileMode::Regular(FileExecution::Executable), + b"120000" | b"160000" => TrackedFileMode::NonRegular, + _ => return Err(SourceStructureError::GitIndexRecord), + }; + let path = admit_inspection_path(&GitPath::new(path.to_vec()))?; + Ok((path.0, mode)) +} + #[cfg(test)] pub(super) fn select_source_paths( present: &BTreeSet, diff --git a/xtask/src/source_structure/tests/replacement.rs b/xtask/src/source_structure/tests/replacement.rs index 40f6d39..2817827 100644 --- a/xtask/src/source_structure/tests/replacement.rs +++ b/xtask/src/source_structure/tests/replacement.rs @@ -28,7 +28,11 @@ fn source_scan_keeps_the_admitted_repository_root() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Date: Tue, 28 Jul 2026 20:38:23 -0700 Subject: [PATCH 100/113] Fix: prebuild smoke fuzz targets --- .github/workflows/ci.yml | 3 +++ CHANGELOG.md | 6 ++++-- xtask/src/fuzz_campaign/workflow_tests.rs | 9 +++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0379ac7..ede3834 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,9 @@ jobs: - name: Prepare deterministic fuzz seeds run: cargo xtask prepare-fuzz-corpus + - name: Build every fuzz target + run: cargo xtask fuzz build --profile smoke + - name: Exercise every fuzz target run: cargo xtask fuzz run --profile smoke diff --git a/CHANGELOG.md b/CHANGELOG.md index a935992..7559c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,8 +74,10 @@ after its public API and format compatibility policies are established. uses a pre-established socket disconnect instead of elapsed-time reachability polling. - Fuzz build and run plans now carry external process deadlines from the - reviewed campaign policy. Run deadlines use checked addition of the - exploration budget and process-grace interval before process-group execution. + reviewed campaign policy. Both smoke and scheduled CI campaigns build every + target under the separate build deadline before applying per-target run + deadlines. Run deadlines use checked addition of the exploration budget and + process-grace interval before process-group execution. - The fuzz dependency-policy gate now grants exact MIT license exceptions to the reviewed `memchr` 2.8.3 and `zmij` 1.0.23 transitive dependencies while retaining Apache-2.0 as the default license allowlist. diff --git a/xtask/src/fuzz_campaign/workflow_tests.rs b/xtask/src/fuzz_campaign/workflow_tests.rs index 76456fe..2cd4883 100644 --- a/xtask/src/fuzz_campaign/workflow_tests.rs +++ b/xtask/src/fuzz_campaign/workflow_tests.rs @@ -75,10 +75,15 @@ fn fuzz_workflows_delegate_campaign_policy_and_execution_to_xtask() -> Result<() assert!(!workflow.contains("python")); assert!(!workflow.contains(".py")); assert!(workflow.contains("cargo xtask fuzz github-env")); - assert!(workflow.contains("cargo xtask fuzz run")); + let build = workflow + .find("cargo xtask fuzz build") + .ok_or("fuzz workflow does not build targets")?; + let run = workflow + .find("cargo xtask fuzz run") + .ok_or("fuzz workflow does not run targets")?; + assert!(build < run); } assert!(SCHEDULED.contains("cargo xtask fuzz check-corpus")); - assert!(SCHEDULED.contains("cargo xtask fuzz build")); assert!(SCHEDULED.contains("cargo xtask fuzz minimize")); Ok(()) } From 4aeefb119a81eb384a9aeffb66858043eb239247 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:40:54 -0700 Subject: [PATCH 101/113] Fix: bound descendant disconnect witness --- .../src/bounded_process/process_group/tests.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 7e7e966..10586ef 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -157,6 +157,7 @@ fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box Result<(), io::Error> { + descendant.set_read_timeout(Some(Duration::from_millis(250)))?; let mut byte = [0_u8; 1]; match io::Read::read_exact(&mut descendant, &mut byte) { Err(source) @@ -174,6 +175,23 @@ fn require_descendant_disconnect(mut descendant: UnixStream) -> Result<(), io::E } } +#[test] +fn descendant_disconnect_witness_has_a_finite_read_deadline() -> Result<(), io::Error> { + let (descendant, _retained_writer) = UnixStream::pair()?; + + let Err(error) = require_descendant_disconnect(descendant) else { + return Err(io::Error::other( + "an open idle witness did not reach its read deadline", + )); + }; + + assert!(matches!( + error.kind(), + io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock + )); + Ok(()) +} + fn wait_for_exit(child: &mut std::process::Child) -> Result { let expires = Instant::now() .checked_add(Duration::from_secs(2)) From 0af1d88ecc4358b24742aa29fc3b507e2d2741f9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:42:57 -0700 Subject: [PATCH 102/113] Fix: preserve closed witness handling --- xtask/src/bounded_process/process_group/tests.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 10586ef..1e5017a 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -157,7 +157,12 @@ fn cleanup_terminates_the_entire_child_process_group() -> Result<(), Box Result<(), io::Error> { - descendant.set_read_timeout(Some(Duration::from_millis(250)))?; + match descendant.set_read_timeout(Some(Duration::from_millis(250))) { + Err(source) if source.kind() == io::ErrorKind::InvalidInput => { + descendant.set_nonblocking(true)?; + } + result => result?, + } let mut byte = [0_u8; 1]; match io::Read::read_exact(&mut descendant, &mut byte) { Err(source) @@ -192,6 +197,14 @@ fn descendant_disconnect_witness_has_a_finite_read_deadline() -> Result<(), io:: Ok(()) } +#[test] +fn descendant_disconnect_witness_accepts_an_already_closed_peer() -> Result<(), io::Error> { + let (descendant, peer) = UnixStream::pair()?; + drop(peer); + + require_descendant_disconnect(descendant) +} + fn wait_for_exit(child: &mut std::process::Child) -> Result { let expires = Instant::now() .checked_add(Duration::from_secs(2)) From c71303f667f656418f0424eb82a0c4bd59c3326f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:43:41 -0700 Subject: [PATCH 103/113] Docs: define corpus revalidation contract --- xtask/src/documentation_integrity/corpus.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index bd17572..aa8390f 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -53,6 +53,19 @@ impl SourceCorpus { Ok(()) } + /// Revalidates every admitted source identity and the exact corpus membership. + /// + /// Each retained source must still match the device, inode, byte length, + /// modification time, and change time admitted for this corpus kind. The + /// method then reruns the bounded Git present/deleted inventories for the + /// same kind and requires the sorted selected path set to remain identical. + /// + /// This operation performs repository metadata I/O and starts bounded Git + /// child processes. It reports path replacement, in-place mutation, added + /// or removed membership, unsafe or non-UTF-8 paths, nonregular sources, and + /// Git or filesystem failures through [`DocumentationError`]. Successful + /// completion confirms that every source and the corpus set remain + /// unchanged; it does not reread source contents. pub(super) fn verify_unchanged( &self, repository_root: &RepositoryRoot, From 5e8a962763e28b6eebe91f813ee77c408964c84f Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 20:45:26 -0700 Subject: [PATCH 104/113] Docs: define internal validation contracts --- xtask/src/documentation_integrity/corpus.rs | 6 ++++++ .../corpus/source_witness.rs | 6 ++++++ xtask/src/documentation_integrity/error.rs | 16 ++++++++++++++ .../error/display/refusal.rs | 10 +++++++++ .../execution/corpus_guard.rs | 21 +++++++++++++++++++ .../execution/snapshot.rs | 17 +++++++++++++++ xtask/src/source_structure/source_error.rs | 5 +++++ xtask/src/source_structure/source_file.rs | 3 +++ 8 files changed, 84 insertions(+) diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index aa8390f..b167b60 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -46,6 +46,12 @@ impl SourceCorpus { &self.paths } + /// Copies every admitted source into its repository-relative snapshot path. + /// + /// Each copy streams from the retained source descriptor, remains within + /// the corpus byte bounds admitted during construction, and revalidates the + /// source identity around the copy. Filesystem or identity failures are + /// returned through [`DocumentationError`]. pub(super) fn materialize(&self, snapshot_root: &Path) -> Result<(), DocumentationError> { for source in &self.sources { source.materialize(snapshot_root, self.kind)?; diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs index e606939..e50a90e 100644 --- a/xtask/src/documentation_integrity/corpus/source_witness.rs +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -42,6 +42,12 @@ impl AdmittedSource { &self.path } + /// Streams the exact admitted bytes into a new read-only snapshot file. + /// + /// The retained descriptor identity is checked before and after copying. + /// The copy uses a fixed 16 KiB buffer and checked offset arithmetic; it + /// performs no durability synchronization because the snapshot is consumed + /// only by the current validation process. pub(super) fn materialize( &self, snapshot_root: &Path, diff --git a/xtask/src/documentation_integrity/error.rs b/xtask/src/documentation_integrity/error.rs index 8ec7f03..25410ed 100644 --- a/xtask/src/documentation_integrity/error.rs +++ b/xtask/src/documentation_integrity/error.rs @@ -16,22 +16,36 @@ pub(crate) enum DocumentationError { first: Box, second: Box, }, + /// One selected source's metadata length exceeds the per-file byte bound. CorpusFileTooLarge { + /// Human-readable corpus kind whose source was refused. corpus: &'static str, + /// Repository-relative source path reported by Git. path: String, + /// Largest admitted metadata length in bytes. maximum: u64, + /// Observed metadata length in bytes. observed: u64, }, + /// Checked aggregate byte accounting overflowed for the named corpus kind. CorpusSizeOverflow(&'static str), + /// The selected corpus's checked aggregate length exceeds its byte bound. CorpusTooLarge { + /// Human-readable corpus kind whose aggregate was refused. corpus: &'static str, + /// Largest admitted aggregate length in bytes. maximum: u64, + /// Observed checked aggregate length in bytes. observed: u64, }, + /// A retained source identity or the exact selected path set changed. CorpusChanged { + /// Human-readable corpus kind whose invariant changed. corpus: &'static str, + /// Repository-relative source path identifying the first difference. path: String, }, + /// Git selected no present sources for the named required corpus kind. EmptyCorpus(&'static str), EnvironmentUnavailable(&'static str), GitInventory(GitInventoryError), @@ -108,7 +122,9 @@ pub(crate) enum DocumentationError { }, /// A filesystem operation could not construct or remove a tool-input snapshot. Snapshot { + /// Snapshot operation that could not complete. action: &'static str, + /// Underlying filesystem failure. source: io::Error, }, VersionMismatch { diff --git a/xtask/src/documentation_integrity/error/display/refusal.rs b/xtask/src/documentation_integrity/error/display/refusal.rs index 2be3ba6..1ee584e 100644 --- a/xtask/src/documentation_integrity/error/display/refusal.rs +++ b/xtask/src/documentation_integrity/error/display/refusal.rs @@ -4,6 +4,11 @@ use std::fmt; use super::super::DocumentationError; +/// Formats one failed filesystem action at the refusal-fixture boundary. +/// +/// `action` names the attempted operation without terminal punctuation. The +/// formatter receives no underlying I/O details; those remain available through +/// the error source chain. pub(super) fn refusal_fixture(formatter: &mut fmt::Formatter<'_>, action: &str) -> fmt::Result { write!( formatter, @@ -11,6 +16,11 @@ pub(super) fn refusal_fixture(formatter: &mut fmt::Formatter<'_>, action: &str) ) } +/// Formats a refusal scenario that did not return its exact reviewed error. +/// +/// The stable prefix identifies `scenario`. When `observed` contains a different +/// typed failure, its escaped display diagnostic is appended as secondary +/// evidence; normal absence records that the malformed input was accepted. pub(super) fn refusal_mismatch( formatter: &mut fmt::Formatter<'_>, scenario: &str, diff --git a/xtask/src/documentation_integrity/execution/corpus_guard.rs b/xtask/src/documentation_integrity/execution/corpus_guard.rs index 1c7c675..40e25e8 100644 --- a/xtask/src/documentation_integrity/execution/corpus_guard.rs +++ b/xtask/src/documentation_integrity/execution/corpus_guard.rs @@ -9,6 +9,17 @@ use crate::repository_file::RepositoryRoot; use super::snapshot::DocumentationSnapshot; use super::{DirectoryToolRunner, DocumentationTool, ToolRunner}; +/// Executes documentation tools against one admitted immutable source snapshot. +/// +/// Construction copies bounded source bytes from retained descriptors and +/// materializes the reviewed repository namespace in a private temporary +/// directory. Every capture revalidates current source identities and exact Git +/// corpus membership before and after the inner runner executes from that +/// snapshot. Post-execution corpus drift takes precedence over a tool result. +/// +/// Construction and capture perform bounded Git child-process and repository +/// I/O. The snapshot is verification evidence only and has no durability role; +/// callers must invoke [`Self::close`] to remove it explicitly. pub(super) struct CorpusGuardedRunner<'a, Runner> { corpora: &'a [&'a SourceCorpus], inner: Runner, @@ -18,6 +29,12 @@ pub(super) struct CorpusGuardedRunner<'a, Runner> { } impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { + /// Builds the source snapshot and binds `inner` to its execution authority. + /// + /// The call allocates bounded path and source inventories, copies admitted + /// bytes, verifies fixed configuration and corpus identities, and returns + /// the exact [`DocumentationError`] from any Git, filesystem, admission, or + /// snapshot failure. pub(super) fn new( inner: Runner, process_directory: &'a RepositoryProcessDirectory, @@ -34,6 +51,10 @@ impl<'a, Runner> CorpusGuardedRunner<'a, Runner> { }) } + /// Releases snapshot descriptors and removes the exact temporary tree. + /// + /// Removal failure is returned as [`DocumentationError::Snapshot`]. This + /// cleanup boundary does not mutate the source repository. pub(super) fn close(self) -> Result<(), DocumentationError> { self.snapshot.close() } diff --git a/xtask/src/documentation_integrity/execution/snapshot.rs b/xtask/src/documentation_integrity/execution/snapshot.rs index d9e07fb..53677f4 100644 --- a/xtask/src/documentation_integrity/execution/snapshot.rs +++ b/xtask/src/documentation_integrity/execution/snapshot.rs @@ -27,6 +27,12 @@ const NAMESPACE_PRESENT: [&str; 5] = [ const NAMESPACE_DELETED: [&str; 3] = ["ls-files", "-z", "--deleted"]; static NEXT_SNAPSHOT: AtomicU64 = AtomicU64::new(0); +/// Private repository-shaped inputs and process authority for documentation tools. +/// +/// Selected sources and the Markdown configuration contain the exact admitted +/// bytes. Other present repository paths are read-only placeholders so offline +/// link validation observes the reviewed namespace without reopening source +/// paths. The owned directory has no durability role and is removed explicitly. pub(super) struct DocumentationSnapshot { directory: SnapshotDirectory, process_directory: RepositoryProcessDirectory, @@ -34,6 +40,11 @@ pub(super) struct DocumentationSnapshot { } impl DocumentationSnapshot { + /// Materializes and revalidates one snapshot from the retained repository. + /// + /// The operation allocates bounded deterministic inventories, performs + /// filesystem writes, and starts bounded Git inventory children. It returns + /// typed corpus, policy-file, Git, or snapshot I/O failures. pub(super) fn create( source_root: &RepositoryRoot, source_process_directory: &RepositoryProcessDirectory, @@ -55,14 +66,20 @@ impl DocumentationSnapshot { }) } + /// Returns the descriptor-backed child working directory for the snapshot. pub(super) const fn process_directory(&self) -> &RepositoryProcessDirectory { &self.process_directory } + /// Returns the capability-relative root used by in-process snapshot readers. pub(super) const fn repository_root(&self) -> &RepositoryRoot { &self.repository_root } + /// Closes owned directory descriptors and removes the exact snapshot tree. + /// + /// Cleanup does not mutate the source repository and reports removal failure + /// through [`DocumentationError::Snapshot`]. pub(super) fn close(self) -> Result<(), DocumentationError> { let Self { directory, diff --git a/xtask/src/source_structure/source_error.rs b/xtask/src/source_structure/source_error.rs index 4992b18..3eba166 100644 --- a/xtask/src/source_structure/source_error.rs +++ b/xtask/src/source_structure/source_error.rs @@ -10,12 +10,17 @@ use crate::diagnostic::{escaped_controls, escaped_path}; use crate::git_inventory::GitInventoryError; pub(crate) enum SourceStructureError { + /// Git's tracked file mode disagrees with the opened worktree object. ExecutionModeChanged { + /// Ambient display path for the disagreed repository entry. path: PathBuf, + /// Canonical tracked mode label admitted from the index. tracked: &'static str, + /// Canonical mode label observed from the opened worktree object. worktree: &'static str, }, GitInventory(GitInventoryError), + /// `git ls-files --stage` returned a malformed, unmerged, or duplicate record. GitIndexRecord, GitPathEncoding { operation: &'static str, diff --git a/xtask/src/source_structure/source_file.rs b/xtask/src/source_structure/source_file.rs index 866e7b1..55dfd5e 100644 --- a/xtask/src/source_structure/source_file.rs +++ b/xtask/src/source_structure/source_file.rs @@ -17,8 +17,11 @@ pub(super) enum FileExecution { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] +/// Canonical Git index mode admitted for one tracked source path. pub(super) enum TrackedFileMode { + /// The index records a symlink or gitlink rather than a regular source. NonRegular, + /// The index records a regular file with the contained executable state. Regular(FileExecution), } From 269a9cca0ae18d99f04409f22fcd6435b4fe3627 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:19:00 -0700 Subject: [PATCH 105/113] Fix: retain fixed policy witnesses --- CHANGELOG.md | 4 +- xtask/src/documentation_integrity.rs | 27 +++++++--- .../contributor_contract.rs | 26 +++++++--- .../src/documentation_integrity/dependabot.rs | 7 +-- .../documentation_integrity/node_toolchain.rs | 9 ++-- .../node_toolchain/tests/lock_graph.rs | 2 +- .../documentation_integrity/policy_corpus.rs | 52 +++++++++++++++++++ xtask/src/documentation_integrity/tests.rs | 49 +++++++++++++++++ .../workflow_contract.rs | 9 ++-- 9 files changed, 159 insertions(+), 26 deletions(-) create mode 100644 xtask/src/documentation_integrity/policy_corpus.rs create mode 100644 xtask/src/documentation_integrity/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7559c81..68bb860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,8 +42,8 @@ after its public API and format compatibility policies are established. membership so newly added sources cannot bypass those tools, executes the tools against a private snapshot copied from the admitted source descriptors so transient path substitution cannot redirect their reads, retains each - fixed policy file identity through semantic admission so a replaced path - cannot validate bytes from a superseded file, + fixed policy file identity through semantic admission and external tool + execution so a replaced path cannot validate bytes from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool execution, and output collection. Validation tools clear the inherited environment and admit only the executable search path and `C` locale, so diff --git a/xtask/src/documentation_integrity.rs b/xtask/src/documentation_integrity.rs index eb6d255..470e7ba 100644 --- a/xtask/src/documentation_integrity.rs +++ b/xtask/src/documentation_integrity.rs @@ -6,13 +6,14 @@ mod dependabot; mod error; mod execution; mod node_toolchain; +mod policy_corpus; mod repository_text; mod tool; mod workflow_contract; use std::path::{Path, PathBuf}; -use crate::repository_file::RepositoryRoot; +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; pub(super) use error::DocumentationError; @@ -22,6 +23,18 @@ pub(super) fn check_refusals() -> Result<(), DocumentationError> { } pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { + check_with(repository_path, execution::run) +} + +fn check_with( + repository_path: &Path, + run_tools: impl FnOnce( + &RepositoryProcessDirectory, + &RepositoryRoot, + &corpus::SourceCorpus, + &corpus::SourceCorpus, + ) -> Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { let repository_root = RepositoryRoot::open(repository_path).map_err(|source| { DocumentationError::RepositoryRootInspect { path: repository_path.to_owned(), @@ -35,13 +48,11 @@ pub(super) fn check(repository_path: &Path) -> Result<(), DocumentationError> { } })?; verify_root(&repository_root, repository_path)?; - contributor_contract::check(&repository_root)?; - node_toolchain::check(&repository_root)?; - dependabot::check(&repository_root, &process_directory)?; - workflow_contract::check(&repository_root)?; + let policies = policy_corpus::FixedPolicyCorpus::admit(&repository_root, &process_directory)?; let markdown = corpus::SourceCorpus::markdown(&repository_root, &process_directory)?; let workflows = corpus::SourceCorpus::workflow(&repository_root, &process_directory)?; - execution::run(&process_directory, &repository_root, &markdown, &workflows)?; + run_tools(&process_directory, &repository_root, &markdown, &workflows)?; + policies.verify(&repository_root)?; verify_root(&repository_root, repository_path) } @@ -60,3 +71,7 @@ fn verify_root( }), } } + +#[cfg(test)] +#[path = "documentation_integrity/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/contributor_contract.rs b/xtask/src/documentation_integrity/contributor_contract.rs index 60f2316..76017fd 100644 --- a/xtask/src/documentation_integrity/contributor_contract.rs +++ b/xtask/src/documentation_integrity/contributor_contract.rs @@ -3,7 +3,7 @@ use crate::repository_file::RepositoryRoot; use super::error::DocumentationError; -use super::repository_text; +use super::repository_text::{self, RepositoryText}; const CONTRIBUTING_PATH: &str = "CONTRIBUTING.md"; const STANDARDS_PATH: &str = "docs/Documentation Standards.md"; @@ -11,13 +11,23 @@ const UNSTAGED_CHECK: &str = "git diff --check"; const STAGED_CHECK: &str = "git diff --cached --check"; const WHOLE_TREE_CHECK: &str = r#"git diff --check "$(git hash-object -t tree /dev/null)" HEAD"#; -pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { - for path in [CONTRIBUTING_PATH, STANDARDS_PATH] { - let raw = repository_text::read(repository_root, path)?; - admit(path, raw.as_str())?; - raw.verify(repository_root)?; - } - Ok(()) +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result<[RepositoryText; 2], DocumentationError> { + Ok([ + checked_source(repository_root, CONTRIBUTING_PATH)?, + checked_source(repository_root, STANDARDS_PATH)?, + ]) +} + +fn checked_source( + repository_root: &RepositoryRoot, + path: &'static str, +) -> Result { + let raw = repository_text::read(repository_root, path)?; + admit(path, raw.as_str())?; + raw.verify(repository_root)?; + Ok(raw) } fn admit(path: &'static str, raw: &str) -> Result<(), DocumentationError> { diff --git a/xtask/src/documentation_integrity/dependabot.rs b/xtask/src/documentation_integrity/dependabot.rs index fcef902..1f6f8e8 100644 --- a/xtask/src/documentation_integrity/dependabot.rs +++ b/xtask/src/documentation_integrity/dependabot.rs @@ -9,7 +9,7 @@ use yaml_rust2::{Yaml, YamlLoader}; use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; use super::error::DocumentationError; -use super::repository_text; +use super::repository_text::{self, RepositoryText}; use manifest::tracked_scopes; const DEPENDABOT_PATH: &str = ".github/dependabot.yml"; @@ -23,11 +23,12 @@ struct DependencyScope { pub(super) fn check( repository_root: &RepositoryRoot, process_directory: &RepositoryProcessDirectory, -) -> Result<(), DocumentationError> { +) -> Result { let raw = repository_text::read(repository_root, DEPENDABOT_PATH)?; let required = tracked_scopes(process_directory)?; admit(raw.as_str(), &required)?; - raw.verify(repository_root) + raw.verify(repository_root)?; + Ok(raw) } fn admit(raw: &str, required: &BTreeSet) -> Result<(), DocumentationError> { diff --git a/xtask/src/documentation_integrity/node_toolchain.rs b/xtask/src/documentation_integrity/node_toolchain.rs index fadf2e7..140a925 100644 --- a/xtask/src/documentation_integrity/node_toolchain.rs +++ b/xtask/src/documentation_integrity/node_toolchain.rs @@ -7,7 +7,7 @@ use serde_json::{Map, Value}; use crate::repository_file::RepositoryRoot; use super::error::DocumentationError; -use super::repository_text; +use super::repository_text::{self, RepositoryText}; const INSTALLER_PATH: &str = "scripts/install_documentation_tools.sh"; const INSTALLER_DIGEST: [u8; 32] = [ @@ -21,7 +21,9 @@ const LOCK_DIGEST: [u8; 32] = [ const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; const MANIFEST_PATH: &str = "scripts/documentation-tools/package.json"; -pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result<[RepositoryText; 3], DocumentationError> { let manifest = repository_text::read(repository_root, MANIFEST_PATH)?; let lock = repository_text::read(repository_root, LOCK_PATH)?; let installer = repository_text::read(repository_root, INSTALLER_PATH)?; @@ -29,7 +31,8 @@ pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), Documentatio admit_lock_bytes(lock.as_str())?; manifest.verify(repository_root)?; lock.verify(repository_root)?; - installer.verify(repository_root) + installer.verify(repository_root)?; + Ok([manifest, lock, installer]) } fn admit(manifest: &str, lock: &str, installer: &str) -> Result<(), DocumentationError> { diff --git a/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs index fd41e4b..3d7515e 100644 --- a/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs +++ b/xtask/src/documentation_integrity/node_toolchain/tests/lock_graph.rs @@ -49,7 +49,7 @@ fn check_with_lock( let repository = TestDirectory::create("node-lock-graph")?; write_repository(&repository, lock)?; let root = RepositoryRoot::open(repository.path())?; - let result = super::super::check(&root); + let result = super::super::check(&root).map(drop); repository.close()?; Ok(result) } diff --git a/xtask/src/documentation_integrity/policy_corpus.rs b/xtask/src/documentation_integrity/policy_corpus.rs new file mode 100644 index 0000000..3384d6c --- /dev/null +++ b/xtask/src/documentation_integrity/policy_corpus.rs @@ -0,0 +1,52 @@ +//! This module owns the retained fixed-policy identity corpus. + +use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; + +use super::contributor_contract; +use super::dependabot; +use super::error::DocumentationError; +use super::node_toolchain; +use super::repository_text::RepositoryText; +use super::workflow_contract; + +const POLICY_SOURCE_COUNT: usize = 7; + +/// The fixed policy files whose admitted identities govern one complete check. +pub(super) struct FixedPolicyCorpus { + sources: [RepositoryText; POLICY_SOURCE_COUNT], +} + +impl FixedPolicyCorpus { + /// Admits every fixed policy file and retains its open identity witness. + pub(super) fn admit( + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + ) -> Result { + let [contributing, standards] = contributor_contract::check(repository_root)?; + let [manifest, lock, installer] = node_toolchain::check(repository_root)?; + let dependabot = dependabot::check(repository_root, process_directory)?; + let workflow = workflow_contract::check(repository_root)?; + Ok(Self { + sources: [ + contributing, + standards, + manifest, + lock, + installer, + dependabot, + workflow, + ], + }) + } + + /// Revalidates every retained policy identity at the final success boundary. + pub(super) fn verify( + &self, + repository_root: &RepositoryRoot, + ) -> Result<(), DocumentationError> { + for source in &self.sources { + source.verify(repository_root)?; + } + Ok(()) + } +} diff --git a/xtask/src/documentation_integrity/tests.rs b/xtask/src/documentation_integrity/tests.rs new file mode 100644 index 0000000..1e78a3b --- /dev/null +++ b/xtask/src/documentation_integrity/tests.rs @@ -0,0 +1,49 @@ +//! This module owns documentation-integrity orchestration regression evidence. + +use std::error::Error; +use std::fs; +use std::io; +use std::path::Path; + +use crate::documentation_integrity::corpus::test_repository::run_git; +use crate::test_directory::TestDirectory; + +use super::{DocumentationError, check_with}; + +const LOCK_PATH: &str = "scripts/documentation-tools/package-lock.json"; + +#[test] +fn fixed_policy_replacement_during_tool_execution_is_refused() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-policy-replacement")?; + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or_else(|| io::Error::other("workspace root is unavailable"))?; + let source = source + .to_str() + .ok_or_else(|| io::Error::other("workspace root is not UTF-8"))?; + run_git( + directory.path(), + &["clone", "--quiet", "--no-hardlinks", source, "repository"], + )?; + let repository = directory.path().join("repository"); + let result = check_with(&repository, |_, _, _, _| replace_lock(&repository)); + assert!(matches!( + result, + Err(DocumentationError::RepositoryFileChanged(LOCK_PATH)) + )); + Ok(()) +} + +fn replace_lock(repository: &Path) -> Result<(), DocumentationError> { + let lock = repository.join(LOCK_PATH); + let replacement = repository.join("replacement-package-lock.json"); + fs::write(&replacement, "{}\n").map_err(inspect)?; + fs::rename(replacement, lock).map_err(inspect) +} + +fn inspect(source: io::Error) -> DocumentationError { + DocumentationError::RepositoryFileInspect { + path: LOCK_PATH, + source, + } +} diff --git a/xtask/src/documentation_integrity/workflow_contract.rs b/xtask/src/documentation_integrity/workflow_contract.rs index 434c2d5..7a59740 100644 --- a/xtask/src/documentation_integrity/workflow_contract.rs +++ b/xtask/src/documentation_integrity/workflow_contract.rs @@ -7,7 +7,7 @@ use yaml_rust2::{Yaml, YamlLoader}; use crate::repository_file::RepositoryRoot; use super::error::DocumentationError; -use super::repository_text; +use super::repository_text::{self, RepositoryText}; use reviewed_step::{DocumentationStep, REVIEWED_STEPS, steps_have_reviewed_membership}; const CI_PATH: &str = ".github/workflows/ci.yml"; @@ -21,10 +21,13 @@ const SETUP_NODE_ACTION: &str = "actions/setup-node@820762786026740c76f36085b0ef const SETUP_NODE_ACTION_PREFIX: &str = "actions/setup-node@"; const NODE_VERSION: &str = "24.18.0"; -pub(super) fn check(repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { +pub(super) fn check( + repository_root: &RepositoryRoot, +) -> Result { let workflow = repository_text::read(repository_root, CI_PATH)?; admit(workflow.as_str())?; - workflow.verify(repository_root) + workflow.verify(repository_root)?; + Ok(workflow) } fn admit(workflow: &str) -> Result<(), DocumentationError> { From 3bb20e8e02e121f459b817c99354e669b4b7fb6c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:22:03 -0700 Subject: [PATCH 106/113] Fix: revalidate corpus identities after inventory --- CHANGELOG.md | 8 +++-- xtask/src/documentation_integrity/corpus.rs | 24 +++++++++++--- .../corpus/replacement_tests.rs | 33 +++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68bb860..25ff975 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,9 +39,11 @@ after its public API and format compatibility policies are established. to 64 MiB before external tools start, retains every selected source identity, refuses device, inode, size, modification-time, or change-time drift before and after each external tool, re-inventories complete corpus - membership so newly added sources cannot bypass those tools, executes the - tools against a private snapshot copied from the admitted source descriptors - so transient path substitution cannot redirect their reads, retains each + membership so newly added sources cannot bypass those tools, revalidates + retained source identities after each rebuilt membership set so same-path + replacement cannot cross the inventory boundary, executes the tools against + a private snapshot copied from the admitted source descriptors so transient + path substitution cannot redirect their reads, retains each fixed policy file identity through semantic admission and external tool execution so a replaced path cannot validate bytes from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index b167b60..dcd9b4f 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -64,7 +64,8 @@ impl SourceCorpus { /// Each retained source must still match the device, inode, byte length, /// modification time, and change time admitted for this corpus kind. The /// method then reruns the bounded Git present/deleted inventories for the - /// same kind and requires the sorted selected path set to remain identical. + /// same kind, requires the sorted selected path set to remain identical, + /// and revalidates every retained identity after re-inventory. /// /// This operation performs repository metadata I/O and starts bounded Git /// child processes. It reports path replacement, in-place mutation, added @@ -77,9 +78,17 @@ impl SourceCorpus { repository_root: &RepositoryRoot, process_directory: &RepositoryProcessDirectory, ) -> Result<(), DocumentationError> { - for source in &self.sources { - source.verify(repository_root, self.kind)?; - } + self.verify_unchanged_with(repository_root, process_directory, || Ok(())) + } + + fn verify_unchanged_with( + &self, + repository_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + after_identity_verification: impl FnOnce() -> Result<(), DocumentationError>, + ) -> Result<(), DocumentationError> { + self.verify_sources(repository_root)?; + after_identity_verification()?; let current = Self::read(repository_root, process_directory, self.kind)?; if let Some(path) = membership_change(&self.paths, ¤t.paths) { return Err(DocumentationError::CorpusChanged { @@ -87,6 +96,13 @@ impl SourceCorpus { path, }); } + self.verify_sources(repository_root) + } + + fn verify_sources(&self, repository_root: &RepositoryRoot) -> Result<(), DocumentationError> { + for source in &self.sources { + source.verify(repository_root, self.kind)?; + } Ok(()) } diff --git a/xtask/src/documentation_integrity/corpus/replacement_tests.rs b/xtask/src/documentation_integrity/corpus/replacement_tests.rs index 695b72a..03c54a2 100644 --- a/xtask/src/documentation_integrity/corpus/replacement_tests.rs +++ b/xtask/src/documentation_integrity/corpus/replacement_tests.rs @@ -60,3 +60,36 @@ fn added_source_refuses_the_admitted_corpus() -> Result<(), Box Result<(), Box> { + let directory = TestDirectory::create("documentation-source-verification-race")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::write(root.join("selected.md"), "# Original\n")?; + fs::write(root.join("substitute.bin"), "# Substitute\n")?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let corpus = SourceCorpus::markdown(&repository_root, &process_directory)?; + + let result = corpus.verify_unchanged_with(&repository_root, &process_directory, || { + fs::rename(root.join("substitute.bin"), root.join("selected.md")).map_err(|source| { + DocumentationError::Snapshot { + action: "replace source between corpus verification phases", + source, + } + }) + }); + + assert!(matches!( + result, + Err(DocumentationError::CorpusChanged { + corpus: "Markdown", + ref path, + }) if path == "selected.md" + )); + drop(corpus); + directory.close()?; + Ok(()) +} From b3c2c4393a89b222f7670f0a163c116d9636b6be Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:29:36 -0700 Subject: [PATCH 107/113] Fix: preserve documentation link targets --- CHANGELOG.md | 9 +- .../corpus/source_witness.rs | 40 +--- .../execution/snapshot.rs | 87 ++------- .../execution/snapshot/namespace.rs | 182 ++++++++++++++++++ .../execution/snapshot/tests.rs | 100 ++++++++++ xtask/src/repository_file.rs | 4 + xtask/src/repository_file/exact_copy.rs | 48 +++++ 7 files changed, 361 insertions(+), 109 deletions(-) create mode 100644 xtask/src/documentation_integrity/execution/snapshot/namespace.rs create mode 100644 xtask/src/documentation_integrity/execution/snapshot/tests.rs create mode 100644 xtask/src/repository_file/exact_copy.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ff975..4ad284c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,9 +43,12 @@ after its public API and format compatibility policies are established. retained source identities after each rebuilt membership set so same-path replacement cannot cross the inventory boundary, executes the tools against a private snapshot copied from the admitted source descriptors so transient - path substitution cannot redirect their reads, retains each - fixed policy file identity through semantic admission and external tool - execution so a replaced path cannot validate bytes from a superseded file, + path substitution cannot redirect their reads, copies bounded regular + non-Markdown namespace files exactly so link-fragment checks observe admitted + target bytes, refuses nonregular namespace targets instead of substituting + placeholders, retains each fixed policy file identity through semantic + admission and external tool execution so a replaced path cannot validate + bytes from a superseded file, and applies a two-minute deadline across Git inventory, validation-tool execution, and output collection. Validation tools clear the inherited environment and admit only the executable search path and `C` locale, so diff --git a/xtask/src/documentation_integrity/corpus/source_witness.rs b/xtask/src/documentation_integrity/corpus/source_witness.rs index e50a90e..40e39f3 100644 --- a/xtask/src/documentation_integrity/corpus/source_witness.rs +++ b/xtask/src/documentation_integrity/corpus/source_witness.rs @@ -2,14 +2,16 @@ use std::fs::File; use std::fs::{self, OpenOptions}; -use std::io::{self, Write}; -use std::os::unix::fs::{FileExt, OpenOptionsExt}; +use std::io; +use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::path::PathBuf; use super::CorpusKind; use crate::documentation_integrity::error::DocumentationError; -use crate::repository_file::{OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot}; +use crate::repository_file::{ + OpenRepositoryFileError, RepositoryFileIdentity, RepositoryRoot, copy_exact, +}; pub(super) struct AdmittedSource { file: File, @@ -115,38 +117,6 @@ impl AdmittedSource { } } -fn copy_exact(source: &File, destination: &mut File, expected: u64) -> Result<(), io::Error> { - let mut offset = 0_u64; - let mut buffer = [0_u8; 16_384]; - while offset < expected { - let remaining = expected - .checked_sub(offset) - .ok_or_else(|| io::Error::other("snapshot source offset exceeded its length"))?; - let limit = - usize::try_from(remaining).map_or(buffer.len(), |bytes| bytes.min(buffer.len())); - let chunk = buffer - .get_mut(..limit) - .ok_or_else(|| io::Error::other("snapshot read bound exceeded its buffer"))?; - let read = source.read_at(chunk, offset)?; - if read == 0 { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "snapshot source ended before its admitted length", - )); - } - let copied = buffer - .get(..read) - .ok_or_else(|| io::Error::other("snapshot write bound exceeded its buffer"))?; - destination.write_all(copied)?; - offset = offset - .checked_add(u64::try_from(read).map_err(|_| { - io::Error::other("snapshot source read length is not representable") - })?) - .ok_or_else(|| io::Error::other("snapshot source offset overflowed"))?; - } - Ok(()) -} - fn identity( file: &File, kind: CorpusKind, diff --git a/xtask/src/documentation_integrity/execution/snapshot.rs b/xtask/src/documentation_integrity/execution/snapshot.rs index 53677f4..1a4987c 100644 --- a/xtask/src/documentation_integrity/execution/snapshot.rs +++ b/xtask/src/documentation_integrity/execution/snapshot.rs @@ -1,5 +1,7 @@ //! This module owns immutable documentation-tool input snapshots. +mod namespace; + use std::collections::BTreeSet; use std::fs::{self, DirBuilder, OpenOptions}; use std::io::{self, Write}; @@ -7,32 +9,22 @@ use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use xtask::protocol_admission::posix_relative_path; - use crate::documentation_integrity::corpus::SourceCorpus; use crate::documentation_integrity::error::DocumentationError; use crate::documentation_integrity::repository_text::{self, RepositoryText}; -use crate::git_inventory::{GitPath, paths_with}; use crate::repository_file::{RepositoryProcessDirectory, RepositoryRoot}; const CREATION_ATTEMPTS: u16 = 1_024; const MARKDOWNLINT_CONFIG: &str = ".markdownlint-cli2.yaml"; -const NAMESPACE_PRESENT: [&str; 5] = [ - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-per-directory=.gitignore", -]; -const NAMESPACE_DELETED: [&str; 3] = ["ls-files", "-z", "--deleted"]; static NEXT_SNAPSHOT: AtomicU64 = AtomicU64::new(0); /// Private repository-shaped inputs and process authority for documentation tools. /// /// Selected sources and the Markdown configuration contain the exact admitted -/// bytes. Other present repository paths are read-only placeholders so offline -/// link validation observes the reviewed namespace without reopening source -/// paths. The owned directory has no durability role and is removed explicitly. +/// bytes. Other representable present regular files are copied exactly through +/// descriptor-bound, identity-checked reads so offline link validation observes +/// faithful target bytes and file types. The owned directory has no durability +/// role and is removed explicitly. pub(super) struct DocumentationSnapshot { directory: SnapshotDirectory, process_directory: RepositoryProcessDirectory, @@ -53,7 +45,12 @@ impl DocumentationSnapshot { let config = repository_text::read(source_root, MARKDOWNLINT_CONFIG)?; let directory = SnapshotDirectory::create()?; let materialized = materialize(directory.path(), corpora, &config)?; - materialize_namespace(directory.path(), source_process_directory, &materialized)?; + namespace::materialize( + directory.path(), + source_root, + source_process_directory, + &materialized, + )?; verify_sources(source_root, source_process_directory, corpora, &config)?; let repository_root = open_snapshot_root(directory.path())?; let process_directory = repository_root.process_directory().map_err(|source| { @@ -124,62 +121,6 @@ fn write_config(destination: &Path, config: &RepositoryText) -> Result<(), Docum .map_err(|source| snapshot_io("write documentation snapshot configuration", source)) } -fn materialize_namespace( - destination: &Path, - process_directory: &RepositoryProcessDirectory, - materialized: &BTreeSet, -) -> Result<(), DocumentationError> { - let present = paths_with( - &NAMESPACE_PRESENT, - "git documentation snapshot present paths", - |command| process_directory.spawn(command), - )?; - let deleted = paths_with( - &NAMESPACE_DELETED, - "git documentation snapshot deleted paths", - |command| process_directory.spawn(command), - )?; - for path in present.difference(&deleted) { - if let Some(relative) = snapshot_relative(path)? - && !materialized.contains(&relative) - { - create_placeholder(destination, &relative)?; - } - } - Ok(()) -} - -fn snapshot_relative(path: &GitPath) -> Result, DocumentationError> { - let Ok(text) = std::str::from_utf8(path.as_bytes()) else { - return Ok(None); - }; - posix_relative_path(text) - .map(Some) - .map_err(|_| DocumentationError::InvalidPath { - corpus: "documentation snapshot namespace", - path: text.to_owned(), - }) -} - -fn create_placeholder(destination: &Path, relative: &Path) -> Result<(), DocumentationError> { - let path = destination.join(relative); - let parent = path.parent().ok_or_else(|| { - snapshot_io( - "resolve documentation snapshot namespace parent", - io::Error::other("namespace path has no parent"), - ) - })?; - fs::create_dir_all(parent) - .map_err(|source| snapshot_io("create documentation snapshot namespace", source))?; - OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o400) - .open(path) - .map(|_| ()) - .map_err(|source| snapshot_io("create documentation snapshot placeholder", source)) -} - fn verify_sources( source_root: &RepositoryRoot, process_directory: &RepositoryProcessDirectory, @@ -264,3 +205,7 @@ fn next_sequence() -> Result { const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { DocumentationError::Snapshot { action, source } } + +#[cfg(test)] +#[path = "snapshot/tests.rs"] +mod tests; diff --git a/xtask/src/documentation_integrity/execution/snapshot/namespace.rs b/xtask/src/documentation_integrity/execution/snapshot/namespace.rs new file mode 100644 index 0000000..4295f34 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot/namespace.rs @@ -0,0 +1,182 @@ +//! This module owns faithful non-corpus snapshot namespace materialization. + +use std::collections::BTreeSet; +use std::fs::{self, File, OpenOptions}; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use xtask::protocol_admission::posix_relative_path; + +use crate::documentation_integrity::error::DocumentationError; +use crate::git_inventory::{GitPath, paths_with}; +use crate::repository_file::{ + OpenRepositoryFileError, RepositoryFileIdentity, RepositoryProcessDirectory, RepositoryRoot, + copy_exact, +}; + +const CORPUS: &str = "documentation snapshot namespace"; +const FILE_LIMIT_BYTES: u64 = 4 * 1_024 * 1_024; +const NAMESPACE_DELETED: [&str; 3] = ["ls-files", "-z", "--deleted"]; +const NAMESPACE_PRESENT: [&str; 5] = [ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-per-directory=.gitignore", +]; +const TOTAL_LIMIT_BYTES: u64 = 64 * 1_024 * 1_024; + +pub(super) fn materialize( + destination: &Path, + source_root: &RepositoryRoot, + process_directory: &RepositoryProcessDirectory, + materialized: &BTreeSet, +) -> Result<(), DocumentationError> { + let present = paths_with( + &NAMESPACE_PRESENT, + "git documentation snapshot present paths", + |command| process_directory.spawn(command), + )?; + let deleted = paths_with( + &NAMESPACE_DELETED, + "git documentation snapshot deleted paths", + |command| process_directory.spawn(command), + )?; + let mut total = 0_u64; + for path in present.difference(&deleted) { + if let Some((relative, text)) = snapshot_relative(path)? + && !materialized.contains(&relative) + { + total = materialize_file(destination, source_root, &relative, &text, total)?; + } + } + Ok(()) +} + +fn materialize_file( + destination: &Path, + source_root: &RepositoryRoot, + relative: &Path, + path: &str, + total: u64, +) -> Result { + let file = open_source(source_root, relative, path)?; + let admitted = identity(&file, path)?; + refuse_file_bound(path, admitted.bytes())?; + let next_total = total + .checked_add(admitted.bytes()) + .ok_or(DocumentationError::CorpusSizeOverflow(CORPUS))?; + refuse_total_bound(next_total)?; + let mut output = create_destination(destination, relative)?; + copy_exact(&file, &mut output, admitted.bytes()) + .map_err(|source| snapshot_io("copy documentation snapshot namespace file", source))?; + verify_identity(&file, source_root, relative, path, &admitted)?; + Ok(next_total) +} + +fn open_source( + source_root: &RepositoryRoot, + relative: &Path, + path: &str, +) -> Result { + match source_root.open_file(relative) { + Ok(file) => Ok(file), + Err(OpenRepositoryFileError::Io(source)) => Err(DocumentationError::Inspect { + corpus: CORPUS, + path: path.to_owned(), + source, + }), + Err(OpenRepositoryFileError::NonRegular) => Err(DocumentationError::NonRegular { + corpus: CORPUS, + path: path.to_owned(), + }), + } +} + +fn create_destination(destination: &Path, relative: &Path) -> Result { + let path = destination.join(relative); + let parent = path.parent().ok_or_else(|| { + snapshot_io( + "resolve documentation snapshot namespace parent", + io::Error::other("namespace path has no parent"), + ) + })?; + fs::create_dir_all(parent) + .map_err(|source| snapshot_io("create documentation snapshot namespace", source))?; + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o400) + .open(path) + .map_err(|source| snapshot_io("create documentation snapshot namespace file", source)) +} + +fn verify_identity( + retained: &File, + source_root: &RepositoryRoot, + relative: &Path, + path: &str, + admitted: &RepositoryFileIdentity, +) -> Result<(), DocumentationError> { + let retained_identity = identity(retained, path)?; + let current = open_source(source_root, relative, path)?; + let current_identity = identity(¤t, path)?; + if &retained_identity == admitted && ¤t_identity == admitted { + Ok(()) + } else { + Err(DocumentationError::CorpusChanged { + corpus: CORPUS, + path: path.to_owned(), + }) + } +} + +fn identity(file: &File, path: &str) -> Result { + RepositoryFileIdentity::read(file).map_err(|source| DocumentationError::Inspect { + corpus: CORPUS, + path: path.to_owned(), + source, + }) +} + +fn refuse_file_bound(path: &str, observed: u64) -> Result<(), DocumentationError> { + if observed <= FILE_LIMIT_BYTES { + Ok(()) + } else { + Err(DocumentationError::CorpusFileTooLarge { + corpus: CORPUS, + path: path.to_owned(), + maximum: FILE_LIMIT_BYTES, + observed, + }) + } +} + +const fn refuse_total_bound(observed: u64) -> Result<(), DocumentationError> { + if observed <= TOTAL_LIMIT_BYTES { + Ok(()) + } else { + Err(DocumentationError::CorpusTooLarge { + corpus: CORPUS, + maximum: TOTAL_LIMIT_BYTES, + observed, + }) + } +} + +fn snapshot_relative(path: &GitPath) -> Result, DocumentationError> { + let Ok(text) = std::str::from_utf8(path.as_bytes()) else { + return Ok(None); + }; + posix_relative_path(text) + .map(|relative| Some((relative, text.to_owned()))) + .map_err(|_| DocumentationError::InvalidPath { + corpus: CORPUS, + path: text.to_owned(), + }) +} + +const fn snapshot_io(action: &'static str, source: io::Error) -> DocumentationError { + DocumentationError::Snapshot { action, source } +} diff --git a/xtask/src/documentation_integrity/execution/snapshot/tests.rs b/xtask/src/documentation_integrity/execution/snapshot/tests.rs new file mode 100644 index 0000000..3b617b5 --- /dev/null +++ b/xtask/src/documentation_integrity/execution/snapshot/tests.rs @@ -0,0 +1,100 @@ +//! This module owns documentation snapshot namespace regression evidence. + +use std::error::Error; +use std::fs; +use std::io::{self, Read}; +use std::os::unix::fs::symlink; +use std::path::Path; + +use crate::documentation_integrity::corpus::{SourceCorpus, test_repository::run_git}; +use crate::repository_file::RepositoryRoot; +use crate::test_directory::TestDirectory; + +use super::DocumentationSnapshot; + +#[test] +fn non_markdown_link_target_preserves_admitted_bytes() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-link-target")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::create_dir(root.join("docs"))?; + fs::write( + root.join("docs/README.md"), + "[target](../target.html#section)\n", + )?; + fs::write(root.join("target.html"), "

Target

\n")?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; + run_git( + root, + &[ + "add", + "--", + ".markdownlint-cli2.yaml", + "docs/README.md", + "target.html", + ], + )?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let markdown = SourceCorpus::markdown(&repository_root, &process_directory)?; + let snapshot = + DocumentationSnapshot::create(&repository_root, &process_directory, &[&markdown])?; + let mut target = snapshot + .repository_root() + .open_file(Path::new("target.html")) + .map_err(|_| io::Error::other("open snapshot target"))?; + let mut observed = Vec::new(); + target.read_to_end(&mut observed)?; + + assert_eq!(observed, b"

Target

\n"); + drop(target); + snapshot.close()?; + drop(markdown); + directory.close()?; + Ok(()) +} + +#[test] +fn nonregular_link_target_is_refused_before_validation() -> Result<(), Box> { + let directory = TestDirectory::create("documentation-symlink-target")?; + let root = directory.path(); + run_git(root, &["init", "--quiet", "--template="])?; + fs::create_dir(root.join("docs"))?; + fs::write(root.join("docs/README.md"), "[target](../target.html)\n")?; + fs::write(root.join("real.html"), "

Target

\n")?; + symlink("real.html", root.join("target.html"))?; + fs::write( + root.join(".markdownlint-cli2.yaml"), + "config:\n MD013: false\n", + )?; + run_git( + root, + &[ + "add", + "--", + ".markdownlint-cli2.yaml", + "docs/README.md", + "real.html", + "target.html", + ], + )?; + let repository_root = RepositoryRoot::open(root)?; + let process_directory = repository_root.process_directory()?; + let markdown = SourceCorpus::markdown(&repository_root, &process_directory)?; + + let result = DocumentationSnapshot::create(&repository_root, &process_directory, &[&markdown]); + + assert!(matches!( + result, + Err(crate::documentation_integrity::DocumentationError::NonRegular { + corpus: "documentation snapshot namespace", + ref path, + }) if path == "target.html" + )); + drop(markdown); + directory.close()?; + Ok(()) +} diff --git a/xtask/src/repository_file.rs b/xtask/src/repository_file.rs index 61f4186..7bdd168 100644 --- a/xtask/src/repository_file.rs +++ b/xtask/src/repository_file.rs @@ -5,6 +5,8 @@ //! replacement cannot silently redirect a read. Supporting another host requires //! an equivalent stable directory-identity contract before enabling these tasks. +mod exact_copy; + use std::fs::File; use std::io; use std::os::fd::{OwnedFd, RawFd}; @@ -17,6 +19,8 @@ use cap_std::ambient_authority; use cap_std::fs::{Dir, OpenOptions}; use repository_process_spawn::set_working_directory; +pub(crate) use exact_copy::copy_exact; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum ReadAccessPolicy { Enabled, diff --git a/xtask/src/repository_file/exact_copy.rs b/xtask/src/repository_file/exact_copy.rs new file mode 100644 index 0000000..c2c815b --- /dev/null +++ b/xtask/src/repository_file/exact_copy.rs @@ -0,0 +1,48 @@ +//! This module owns bounded exact descriptor-to-file copying. + +use std::fs::File; +use std::io::{self, Write}; +use std::os::unix::fs::FileExt; + +const COPY_BUFFER_BYTES: usize = 16_384; + +/// Copies exactly the admitted byte length from a descriptor at offset zero. +/// +/// The fixed-size buffer bounds memory use. Short sources, offset overflow, +/// unrepresentable read lengths, and destination write failures are refused. +pub(crate) fn copy_exact( + source: &File, + destination: &mut File, + expected: u64, +) -> Result<(), io::Error> { + let mut offset = 0_u64; + let mut buffer = [0_u8; COPY_BUFFER_BYTES]; + while offset < expected { + let remaining = expected + .checked_sub(offset) + .ok_or_else(|| io::Error::other("source offset exceeded its admitted length"))?; + let limit = + usize::try_from(remaining).map_or(buffer.len(), |bytes| bytes.min(buffer.len())); + let chunk = buffer + .get_mut(..limit) + .ok_or_else(|| io::Error::other("read bound exceeded the copy buffer"))?; + let read = source.read_at(chunk, offset)?; + if read == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "source ended before its admitted length", + )); + } + let copied = buffer + .get(..read) + .ok_or_else(|| io::Error::other("write bound exceeded the copy buffer"))?; + destination.write_all(copied)?; + offset = offset + .checked_add( + u64::try_from(read) + .map_err(|_| io::Error::other("source read length is not representable"))?, + ) + .ok_or_else(|| io::Error::other("source offset overflowed"))?; + } + Ok(()) +} From 35195e5b9036eeafcd7af30f5245ed199c6d5fef Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:31:30 -0700 Subject: [PATCH 108/113] Fix: retain snapshot cleanup failures --- CHANGELOG.md | 5 ++- .../src/documentation_integrity/execution.rs | 14 +++++- .../execution/finish_tests.rs | 43 +++++++++++++++++++ 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 xtask/src/documentation_integrity/execution/finish_tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad284c..6eb63ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,9 @@ after its public API and format compatibility policies are established. boundary, and the seven superseded Python checkers have been removed. The boundary rejects duplicate repository JSON fields and unlocked installer substitutions, admits only the exact reviewed Node lock artifact, retains - simultaneous Markdown and link failures, parses documentation workflow - commands as YAML, rejects guarded or non-string `run` values, preserves + simultaneous Markdown and link failures, retains both the primary tool + failure and a simultaneous snapshot-cleanup failure, parses documentation + workflow commands as YAML, rejects guarded or non-string `run` values, preserves declarations after Dependabot directory lists, compares Dependabot maintenance fields as typed YAML values, requires every reviewed documentation CI command and the pinned Node setup action exactly once, diff --git a/xtask/src/documentation_integrity/execution.rs b/xtask/src/documentation_integrity/execution.rs index 6257758..b2aff9a 100644 --- a/xtask/src/documentation_integrity/execution.rs +++ b/xtask/src/documentation_integrity/execution.rs @@ -53,8 +53,14 @@ pub(super) fn run( CorpusGuardedRunner::new(external, process_directory, repository_root, &corpora)?; let result = run_with(&mut runner, markdown.paths(), workflows.paths()); let cleanup = runner.close(); - result?; - cleanup + finish_run(result, cleanup) +} + +fn finish_run( + result: Result<(), DocumentationError>, + cleanup: Result<(), DocumentationError>, +) -> Result<(), DocumentationError> { + combine_checks(result, cleanup) } /// Executes both named malformed-input scenarios through the production runner. @@ -225,3 +231,7 @@ fn documentation_command(tool: DocumentationTool, arguments: &[String], path: &O #[cfg(test)] #[path = "execution/tests.rs"] mod tests; + +#[cfg(test)] +#[path = "execution/finish_tests.rs"] +mod finish_tests; diff --git a/xtask/src/documentation_integrity/execution/finish_tests.rs b/xtask/src/documentation_integrity/execution/finish_tests.rs new file mode 100644 index 0000000..b94cfdf --- /dev/null +++ b/xtask/src/documentation_integrity/execution/finish_tests.rs @@ -0,0 +1,43 @@ +//! This module owns documentation-run closure regression evidence. + +use std::io; + +use crate::documentation_integrity::DocumentationError; + +use super::finish_run; + +#[test] +fn tool_and_snapshot_cleanup_failures_are_both_reported() { + let result = finish_run( + Err(DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + stdout: String::from("tool failure"), + stderr: String::new(), + }), + Err(DocumentationError::Snapshot { + action: "remove documentation snapshot", + source: io::Error::other("cleanup failure"), + }), + ); + + assert!(matches!( + result, + Err(DocumentationError::CheckFailures { first, second }) + if matches!( + *first, + DocumentationError::ToolFailed { + program: "markdownlint-cli2", + code: Some(1), + ref stdout, + ref stderr, + } if stdout == "tool failure" && stderr.is_empty() + ) && matches!( + *second, + DocumentationError::Snapshot { + action: "remove documentation snapshot", + .. + } + ) + )); +} From 5b22aed31cbb20b5cf89d1fe8223bb0338574563 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:32:51 -0700 Subject: [PATCH 109/113] Test: lock duplicate YAML key refusal --- CHANGELOG.md | 5 +++-- .../dependabot/tests.rs | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eb63ee..9d3de08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,9 @@ after its public API and format compatibility policies are established. substitutions, admits only the exact reviewed Node lock artifact, retains simultaneous Markdown and link failures, retains both the primary tool failure and a simultaneous snapshot-cleanup failure, parses documentation - workflow commands as YAML, rejects guarded or non-string `run` values, preserves - declarations after Dependabot directory lists, compares Dependabot + workflow commands as YAML, rejects guarded or non-string `run` values, + preserves declarations after Dependabot directory lists, refuses duplicate + Dependabot YAML mapping keys before semantic admission, compares Dependabot maintenance fields as typed YAML values, requires every reviewed documentation CI command and the pinned Node setup action exactly once, admits only the exact pinned checkout and Node setup actions, requires actions diff --git a/xtask/src/documentation_integrity/dependabot/tests.rs b/xtask/src/documentation_integrity/dependabot/tests.rs index c5cf8ce..28b6334 100644 --- a/xtask/src/documentation_integrity/dependabot/tests.rs +++ b/xtask/src/documentation_integrity/dependabot/tests.rs @@ -84,6 +84,25 @@ fn duplicate_update_scope_is_refused() { )); } +#[test] +fn duplicate_yaml_mapping_keys_are_refused_before_policy_admission() { + let top_level = format!("{POLICY}updates: []\n"); + let nested = POLICY.replacen( + " interval: weekly\n", + " interval: weekly\n interval: daily\n", + 1, + ); + for policy in [top_level, nested] { + assert!(matches!( + super::admit(&policy, &required()), + Err(super::DocumentationError::RepositoryYaml { + path: super::DEPENDABOT_PATH, + .. + }) + )); + } +} + #[test] fn nonuniform_maintenance_policy_is_refused() { let policy = POLICY.replacen(" interval: weekly", " interval: daily", 1); From 8ce0b835399efd5b29edc99b30cdcf77c915dd12 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:35:42 -0700 Subject: [PATCH 110/113] Fix: parse executable fuzz workflow steps --- CHANGELOG.md | 7 +- xtask/src/fuzz_campaign/workflow_tests.rs | 106 ++++++++++++++++++---- 2 files changed, 92 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d3de08..cb6ac7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,8 +83,11 @@ after its public API and format compatibility policies are established. - Fuzz build and run plans now carry external process deadlines from the reviewed campaign policy. Both smoke and scheduled CI campaigns build every target under the separate build deadline before applying per-target run - deadlines. Run deadlines use checked addition of the exploration budget and - process-grace interval before process-group execution. + deadlines. Workflow contract evidence parses only executable `run` scalars in + the reviewed fuzz jobs, so comments, names, and environment values cannot + impersonate required build or run commands. Run deadlines use checked + addition of the exploration budget and process-grace interval before + process-group execution. - The fuzz dependency-policy gate now grants exact MIT license exceptions to the reviewed `memchr` 2.8.3 and `zmij` 1.0.23 transitive dependencies while retaining Apache-2.0 as the default license allowlist. diff --git a/xtask/src/fuzz_campaign/workflow_tests.rs b/xtask/src/fuzz_campaign/workflow_tests.rs index 2cd4883..8e776da 100644 --- a/xtask/src/fuzz_campaign/workflow_tests.rs +++ b/xtask/src/fuzz_campaign/workflow_tests.rs @@ -3,6 +3,8 @@ use std::error::Error; use std::fs; use std::path::Path; +use yaml_rust2::{Yaml, YamlLoader}; + const CI: &str = include_str!("../../../.github/workflows/ci.yml"); const SCHEDULED: &str = include_str!("../../../.github/workflows/fuzz-scheduled.yml"); @@ -65,29 +67,95 @@ fn only_successfully_minimized_corpora_are_retained() -> Result<(), Box Result<(), Box> { - let (_, ci_fuzz) = CI - .split_once(" fuzz-smoke:\n") - .ok_or("CI has no fuzz-smoke job")?; - let (ci_fuzz, _) = ci_fuzz - .split_once("\n dependency-policy:") - .ok_or("CI fuzz-smoke job has no closing job")?; - for workflow in [ci_fuzz, SCHEDULED] { - assert!(!workflow.contains("python")); - assert!(!workflow.contains(".py")); - assert!(workflow.contains("cargo xtask fuzz github-env")); - let build = workflow - .find("cargo xtask fuzz build") - .ok_or("fuzz workflow does not build targets")?; - let run = workflow - .find("cargo xtask fuzz run") - .ok_or("fuzz workflow does not run targets")?; - assert!(build < run); + for (workflow, job) in [(CI, "fuzz-smoke"), (SCHEDULED, "fuzz")] { + workflow_delegates_to_xtask(workflow, job)?; + } + let scheduled_commands = run_commands(SCHEDULED, "fuzz")?; + if command_index(&scheduled_commands, "cargo xtask fuzz check-corpus").is_none() { + return Err("scheduled workflow does not validate its corpus".into()); + } + if command_index(&scheduled_commands, "cargo xtask fuzz minimize").is_none() { + return Err("scheduled workflow does not minimize its corpus".into()); } - assert!(SCHEDULED.contains("cargo xtask fuzz check-corpus")); - assert!(SCHEDULED.contains("cargo xtask fuzz minimize")); Ok(()) } +#[test] +fn non_run_scalar_cannot_impersonate_the_fuzz_build_step() { + let disguised = CI.replace( + " run: cargo xtask fuzz build --profile smoke", + " env:\n BUILD_NOTE: cargo xtask fuzz build --profile smoke", + ); + + assert!(workflow_delegates_to_xtask(&disguised, "fuzz-smoke").is_err()); +} + +fn workflow_delegates_to_xtask(workflow: &str, job: &str) -> Result<(), Box> { + let commands = run_commands(workflow, job)?; + if commands + .iter() + .any(|command| command.contains("python") || command.contains(".py")) + { + return Err("fuzz workflow delegates to Python".into()); + } + if command_index(&commands, "cargo xtask fuzz github-env").is_none() { + return Err("fuzz workflow does not load xtask policy".into()); + } + let build = command_index(&commands, "cargo xtask fuzz build") + .ok_or("fuzz workflow does not build targets")?; + let run = command_index(&commands, "cargo xtask fuzz run") + .ok_or("fuzz workflow does not run targets")?; + if build >= run { + return Err("fuzz workflow does not build before running targets".into()); + } + Ok(()) +} + +fn run_commands(workflow: &str, job: &str) -> Result, Box> { + let documents = YamlLoader::load_from_str(workflow)?; + let [document] = documents.as_slice() else { + return Err("fuzz workflow must contain exactly one YAML document".into()); + }; + let jobs = mapping_field(document, "jobs") + .and_then(Yaml::as_hash) + .ok_or("fuzz workflow jobs must be a mapping")?; + let job = jobs + .get(&Yaml::String(job.to_owned())) + .ok_or("fuzz workflow has no reviewed fuzz job")?; + let steps = mapping_field(job, "steps") + .and_then(Yaml::as_vec) + .ok_or("fuzz workflow job steps must be a sequence")?; + let mut commands = Vec::new(); + for step in steps { + let Some(run) = mapping_field(step, "run") else { + continue; + }; + let run = run + .as_str() + .ok_or("fuzz workflow run step must be a string")?; + commands.push(run.to_owned()); + } + Ok(commands) +} + +fn mapping_field<'a>(node: &'a Yaml, field: &str) -> Option<&'a Yaml> { + node.as_hash()?.get(&Yaml::String(field.to_owned())) +} + +fn command_index(commands: &[String], expected: &str) -> Option { + commands.iter().position(|command| { + command.lines().any(|line| { + line.trim().strip_prefix(expected).is_some_and(|suffix| { + suffix.is_empty() + || suffix + .as_bytes() + .first() + .is_some_and(u8::is_ascii_whitespace) + }) + }) + }) +} + fn checkout_references(workflow: &str) -> BTreeSet<&str> { action_references(workflow) .filter_map(|(action, reference)| (action == "actions/checkout").then_some(reference)) From 47654467a8a69f8c530a83d4084728d488f2e520 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:37:13 -0700 Subject: [PATCH 111/113] Refactor: centralize regular source admission --- xtask/src/source_structure.rs | 29 ++++++++++++++------------- xtask/tests/source_policy_contract.rs | 11 ++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/xtask/src/source_structure.rs b/xtask/src/source_structure.rs index 7b485a5..4f1ffa4 100644 --- a/xtask/src/source_structure.rs +++ b/xtask/src/source_structure.rs @@ -80,13 +80,7 @@ fn inventory_violations( let mut violations = Vec::new(); for relative in executable_candidates { let tracked_mode = tracked_modes.get(relative.as_path()).copied(); - let SourceFileAdmission::Regular(source) = - AdmittedSource::admit(source_root, relative.as_path(), tracked_mode)? - else { - return Err(SourceStructureError::NonRegular( - source_root.display_path(relative.as_path()), - )); - }; + let source = admit_regular(source_root, relative.as_path(), tracked_mode)?; if source.execution() == FileExecution::Executable && source_line_count(source_root, &source)? == SourceLineCount::Exceeded { @@ -118,13 +112,7 @@ fn source_violations_with_modes( let mut violations = Vec::new(); for relative in paths { let tracked_mode = tracked_modes.get(relative.as_path()).copied(); - let SourceFileAdmission::Regular(source) = - AdmittedSource::admit(source_root, relative.as_path(), tracked_mode)? - else { - return Err(SourceStructureError::NonRegular( - source_root.display_path(relative.as_path()), - )); - }; + let source = admit_regular(source_root, relative.as_path(), tracked_mode)?; if is_extensionless_file(relative.as_str().as_bytes()) && source.execution() == FileExecution::NonExecutable { @@ -138,6 +126,19 @@ fn source_violations_with_modes( Ok(violations) } +fn admit_regular( + source_root: &RepositoryRoot, + relative: &Path, + tracked_mode: Option, +) -> Result { + match AdmittedSource::admit(source_root, relative, tracked_mode)? { + SourceFileAdmission::Regular(source) => Ok(source), + SourceFileAdmission::NonRegular => Err(SourceStructureError::NonRegular( + source_root.display_path(relative), + )), + } +} + fn source_line_count( source_root: &RepositoryRoot, source: &AdmittedSource, diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index 092cc53..f6d4e96 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -55,6 +55,17 @@ fn source_inventory_uses_the_admitted_repository_directory() { assert!(!GIT_PROCESS.contains("current_dir(")); } +#[test] +fn regular_source_admission_has_one_refusal_boundary() { + assert_eq!( + SOURCE_STRUCTURE + .matches("SourceFileAdmission::NonRegular") + .count(), + 1 + ); + assert_eq!(SOURCE_STRUCTURE.matches("fn admit_regular(").count(), 1); +} + #[test] fn git_inventory_uses_the_deadline_bounded_process_layer() { assert!(GIT_PROCESS.contains("const GIT_DEADLINE: Duration")); From 4bf791c19f17670ebc0eec2a89fcd805c70914d9 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:42:13 -0700 Subject: [PATCH 112/113] Refactor: centralize Git fixture execution --- xtask/src/documentation_integrity/corpus.rs | 2 -- .../corpus/replacement_tests.rs | 3 +- .../documentation_integrity/corpus/tests.rs | 3 +- .../execution/snapshot/tests.rs | 3 +- .../execution/tests.rs | 3 +- xtask/src/documentation_integrity/tests.rs | 2 +- xtask/src/main.rs | 6 ++++ ...st_repository.rs => repository_fixture.rs} | 10 +++--- xtask/src/source_structure/pure_rust_tests.rs | 33 +----------------- xtask/tests/source_policy_contract.rs | 34 +++++++++++-------- 10 files changed, 39 insertions(+), 60 deletions(-) rename xtask/src/{documentation_integrity/corpus/test_repository.rs => repository_fixture.rs} (86%) diff --git a/xtask/src/documentation_integrity/corpus.rs b/xtask/src/documentation_integrity/corpus.rs index dcd9b4f..df3bbb3 100644 --- a/xtask/src/documentation_integrity/corpus.rs +++ b/xtask/src/documentation_integrity/corpus.rs @@ -3,8 +3,6 @@ mod byte_budget; mod selection; mod source_witness; -#[cfg(test)] -pub(super) mod test_repository; use std::cmp::Ordering; use std::io; diff --git a/xtask/src/documentation_integrity/corpus/replacement_tests.rs b/xtask/src/documentation_integrity/corpus/replacement_tests.rs index 03c54a2..53bd07a 100644 --- a/xtask/src/documentation_integrity/corpus/replacement_tests.rs +++ b/xtask/src/documentation_integrity/corpus/replacement_tests.rs @@ -2,9 +2,10 @@ use std::fs; -use super::{SourceCorpus, test_repository::run_git}; +use super::SourceCorpus; use crate::documentation_integrity::error::DocumentationError; use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; #[test] diff --git a/xtask/src/documentation_integrity/corpus/tests.rs b/xtask/src/documentation_integrity/corpus/tests.rs index 893c12a..82e43c2 100644 --- a/xtask/src/documentation_integrity/corpus/tests.rs +++ b/xtask/src/documentation_integrity/corpus/tests.rs @@ -5,10 +5,11 @@ use std::fs; use std::path::Path; use std::process::Command; -use super::{CorpusKind, SourceCorpus, admit_path, test_repository::run_git}; +use super::{CorpusKind, SourceCorpus, admit_path}; use crate::documentation_integrity::error::DocumentationError; use crate::git_inventory::GitPath; use crate::repository_file::{OpenRepositoryFileError, RepositoryRoot}; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; #[test] diff --git a/xtask/src/documentation_integrity/execution/snapshot/tests.rs b/xtask/src/documentation_integrity/execution/snapshot/tests.rs index 3b617b5..3f3cfad 100644 --- a/xtask/src/documentation_integrity/execution/snapshot/tests.rs +++ b/xtask/src/documentation_integrity/execution/snapshot/tests.rs @@ -6,8 +6,9 @@ use std::io::{self, Read}; use std::os::unix::fs::symlink; use std::path::Path; -use crate::documentation_integrity::corpus::{SourceCorpus, test_repository::run_git}; +use crate::documentation_integrity::corpus::SourceCorpus; use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; use super::DocumentationSnapshot; diff --git a/xtask/src/documentation_integrity/execution/tests.rs b/xtask/src/documentation_integrity/execution/tests.rs index 2f171eb..3f68f85 100644 --- a/xtask/src/documentation_integrity/execution/tests.rs +++ b/xtask/src/documentation_integrity/execution/tests.rs @@ -5,8 +5,9 @@ use std::io::Read; use std::path::{Path, PathBuf}; use crate::bounded_process::ProcessOutput; -use crate::documentation_integrity::corpus::{SourceCorpus, test_repository::run_git}; +use crate::documentation_integrity::corpus::SourceCorpus; use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; use super::corpus_guard::CorpusGuardedRunner; diff --git a/xtask/src/documentation_integrity/tests.rs b/xtask/src/documentation_integrity/tests.rs index 1e78a3b..2019568 100644 --- a/xtask/src/documentation_integrity/tests.rs +++ b/xtask/src/documentation_integrity/tests.rs @@ -5,7 +5,7 @@ use std::fs; use std::io; use std::path::Path; -use crate::documentation_integrity::corpus::test_repository::run_git; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; use super::{DocumentationError, check_with}; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c167b41..a6d4358 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -54,6 +54,12 @@ mod protocol_conformance; reason = "repository file admission is shared by sibling repository tasks" )] mod repository_file; +#[cfg(test)] +#[allow( + clippy::redundant_pub_crate, + reason = "repository fixture execution is shared by sibling test modules" +)] +mod repository_fixture; #[allow( clippy::redundant_pub_crate, reason = "the parent command dispatcher is the only consumer" diff --git a/xtask/src/documentation_integrity/corpus/test_repository.rs b/xtask/src/repository_fixture.rs similarity index 86% rename from xtask/src/documentation_integrity/corpus/test_repository.rs rename to xtask/src/repository_fixture.rs index 2b8c9d9..3559690 100644 --- a/xtask/src/documentation_integrity/corpus/test_repository.rs +++ b/xtask/src/repository_fixture.rs @@ -1,4 +1,4 @@ -//! This module owns hermetic Git commands for corpus regression repositories. +//! This module owns hermetic Git commands for repository regression fixtures. use std::env; use std::error::Error; @@ -19,10 +19,7 @@ const GIT_FIXTURE_DEADLINE: Duration = Duration::from_mins(2); /// path, deterministic locale, and null system and global Git configuration. /// Process failures retain their typed source; a nonzero Git status reports the /// attempted arguments without admitting tool output. -pub(in crate::documentation_integrity) fn run_git( - root: &Path, - arguments: &[&str], -) -> Result<(), Box> { +pub(crate) fn run_git(root: &Path, arguments: &[&str]) -> Result<(), Box> { let path = env::var_os("PATH") .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "PATH is unavailable"))?; let mut command = Command::new("git"); @@ -35,10 +32,11 @@ pub(in crate::documentation_integrity) fn run_git( .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_COUNT", "0") .env("LC_ALL", "C") + .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); let output = bounded_process::status( - "documentation Git fixture", + "repository Git fixture", &mut command, Some(GIT_FIXTURE_DEADLINE), )?; diff --git a/xtask/src/source_structure/pure_rust_tests.rs b/xtask/src/source_structure/pure_rust_tests.rs index 08e56a1..16cc1a4 100644 --- a/xtask/src/source_structure/pure_rust_tests.rs +++ b/xtask/src/source_structure/pure_rust_tests.rs @@ -4,12 +4,10 @@ use std::collections::BTreeSet; use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::time::Duration; -use crate::bounded_process; use crate::git_inventory::GitPath; use crate::repository_file::RepositoryRoot; +use crate::repository_fixture::run_git; use crate::test_directory::TestDirectory; #[test] @@ -189,32 +187,3 @@ impl SourceFixture { Ok(()) } } - -fn run_git( - repository: &std::path::Path, - arguments: &[&str], -) -> Result<(), Box> { - let path = std::env::var_os("PATH").ok_or("test PATH is unavailable")?; - let mut command = Command::new("git"); - command - .args(arguments) - .current_dir(repository) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .env_clear() - .env("PATH", path) - .env("LC_ALL", "C") - .env("GIT_CONFIG_NOSYSTEM", "1") - .env("GIT_CONFIG_GLOBAL", "/dev/null"); - let output = bounded_process::status( - "git test fixture", - &mut command, - Some(Duration::from_secs(10)), - )?; - if output.succeeded { - Ok(()) - } else { - Err(format!("git test fixture failed with status {:?}", output.code).into()) - } -} diff --git a/xtask/tests/source_policy_contract.rs b/xtask/tests/source_policy_contract.rs index f6d4e96..f1d9efb 100644 --- a/xtask/tests/source_policy_contract.rs +++ b/xtask/tests/source_policy_contract.rs @@ -11,12 +11,12 @@ const BOUNDED_PROCESS_GROUP_TESTS: &str = include_str!("../src/bounded_process/process_group/tests.rs"); const BOUNDED_PROCESS_READER: &str = include_str!("../src/bounded_process/reader.rs"); const BOUNDED_PROCESS_TESTS: &str = include_str!("../src/bounded_process/tests.rs"); -const DOCUMENTATION_TEST_REPOSITORY: &str = - include_str!("../src/documentation_integrity/corpus/test_repository.rs"); +const REPOSITORY_FIXTURE: &str = include_str!("../src/repository_fixture.rs"); const GIT_INVENTORY_ERROR: &str = include_str!("../src/git_inventory/error.rs"); const GIT_PATH_STREAM: &str = include_str!("../src/git_inventory/path_stream.rs"); const GIT_PROCESS: &str = include_str!("../src/git_inventory/process.rs"); const REPOSITORY_FILE: &str = include_str!("../src/repository_file.rs"); +const SOURCE_PURE_RUST_TESTS: &str = include_str!("../src/source_structure/pure_rust_tests.rs"); const SOURCE_STRUCTURE: &str = include_str!("../src/source_structure.rs"); const SOURCE_INVENTORY: &str = include_str!("../src/source_structure/source_inventory.rs"); @@ -66,6 +66,13 @@ fn regular_source_admission_has_one_refusal_boundary() { assert_eq!(SOURCE_STRUCTURE.matches("fn admit_regular(").count(), 1); } +#[test] +fn sanitized_git_fixture_has_one_process_authority() { + let definitions = REPOSITORY_FIXTURE.matches("fn run_git(").count() + + SOURCE_PURE_RUST_TESTS.matches("fn run_git(").count(); + assert_eq!(definitions, 1); +} + #[test] fn git_inventory_uses_the_deadline_bounded_process_layer() { assert!(GIT_PROCESS.contains("const GIT_DEADLINE: Duration")); @@ -114,18 +121,18 @@ fn descendant_cleanup_uses_disconnect_evidence_instead_of_elapsed_time() { } #[test] -fn documentation_git_fixtures_use_the_bounded_process_layer() { - assert!(DOCUMENTATION_TEST_REPOSITORY.contains("bounded_process::status(")); - assert!(DOCUMENTATION_TEST_REPOSITORY.contains("GIT_FIXTURE_DEADLINE")); - assert!(!DOCUMENTATION_TEST_REPOSITORY.contains(".output()")); +fn repository_git_fixtures_use_the_bounded_process_layer() { + assert!(REPOSITORY_FIXTURE.contains("bounded_process::status(")); + assert!(REPOSITORY_FIXTURE.contains("GIT_FIXTURE_DEADLINE")); + assert!(!REPOSITORY_FIXTURE.contains(".output()")); } #[test] -fn documentation_git_fixtures_clear_the_ambient_environment() { - assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env_clear()")); - assert!(DOCUMENTATION_TEST_REPOSITORY.contains("env::var_os(\"PATH\")")); - assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env(\"PATH\"")); - assert!(DOCUMENTATION_TEST_REPOSITORY.contains(".env(\"LC_ALL\", \"C\")")); +fn repository_git_fixtures_clear_the_ambient_environment() { + assert!(REPOSITORY_FIXTURE.contains(".env_clear()")); + assert!(REPOSITORY_FIXTURE.contains("env::var_os(\"PATH\")")); + assert!(REPOSITORY_FIXTURE.contains(".env(\"PATH\"")); + assert!(REPOSITORY_FIXTURE.contains(".env(\"LC_ALL\", \"C\")")); } #[test] @@ -210,10 +217,7 @@ fn repository_process_boundaries_document_every_exported_contract() -> Result<() " pub(super) fn join(", ], )?; - require_docs( - DOCUMENTATION_TEST_REPOSITORY, - &["pub(in crate::documentation_integrity) fn run_git("], - )?; + require_docs(REPOSITORY_FIXTURE, &["pub(crate) fn run_git("])?; require_docs( GIT_INVENTORY_ERROR, &[ From 1254cdc78c67425c29cee12588e3e64afed5ed26 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 28 Jul 2026 21:48:16 -0700 Subject: [PATCH 113/113] Docs: align fixture process contract --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6ac7d..e9029eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,8 +64,8 @@ after its public API and format compatibility policies are established. Git-backed process fixtures clear the inherited environment, explicitly admit the executable search path and `C` locale, ignore system and global Git configuration, and preserve non-UTF-8 template paths without lossy - conversion. Documentation corpus fixture commands also use the bounded - process authority with dedicated groups, null output, and a two-minute + conversion. Repository-backed Git fixtures share one bounded process + authority with dedicated groups, null standard streams, and a two-minute deadline. Documentation Git inventory and tools start from one retained repository directory handle, so transient replacement of the ambient repository path cannot redirect validation. Retained and per-spawn directory